2024-11-17 01:51:26,849 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-17 01:51:26,864 main DEBUG Took 0.012601 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-17 01:51:26,864 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-17 01:51:26,864 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-17 01:51:26,866 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-17 01:51:26,867 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:51:26,876 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-17 01:51:26,889 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:51:26,891 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:51:26,892 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:51:26,892 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:51:26,893 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:51:26,893 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:51:26,895 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:51:26,895 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:51:26,896 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:51:26,896 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:51:26,898 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:51:26,898 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:51:26,899 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:51:26,899 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:51:26,900 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:51:26,900 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:51:26,901 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:51:26,901 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:51:26,902 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:51:26,902 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:51:26,903 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:51:26,903 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:51:26,904 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:51:26,904 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:51:26,905 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:51:26,905 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-17 01:51:26,907 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:51:26,909 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-17 01:51:26,911 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-17 01:51:26,912 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-17 01:51:26,913 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-17 01:51:26,914 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-17 01:51:26,924 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-17 01:51:26,928 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-17 01:51:26,930 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-17 01:51:26,931 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-17 01:51:26,931 main DEBUG createAppenders(={Console}) 2024-11-17 01:51:26,932 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-17 01:51:26,933 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-17 01:51:26,933 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-17 01:51:26,934 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-17 01:51:26,934 main DEBUG OutputStream closed 2024-11-17 01:51:26,934 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-17 01:51:26,935 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-17 01:51:26,935 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-17 01:51:27,011 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-17 01:51:27,013 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-17 01:51:27,014 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-17 01:51:27,015 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-17 01:51:27,015 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-17 01:51:27,016 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-17 01:51:27,016 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-17 01:51:27,016 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-17 01:51:27,016 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-17 01:51:27,017 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-17 01:51:27,017 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-17 01:51:27,017 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-17 01:51:27,018 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-17 01:51:27,018 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-17 01:51:27,018 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-17 01:51:27,018 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-17 01:51:27,019 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-17 01:51:27,019 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-17 01:51:27,021 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17 01:51:27,022 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-17 01:51:27,022 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-17 01:51:27,023 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-17T01:51:27,232 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8 2024-11-17 01:51:27,235 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-17 01:51:27,235 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17T01:51:27,243 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-17T01:51:27,264 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-17T01:51:27,279 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/cluster_97028d77-d23a-8219-e606-b0c441f49b52, deleteOnExit=true 2024-11-17T01:51:27,279 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-17T01:51:27,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/test.cache.data in system properties and HBase conf 2024-11-17T01:51:27,281 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/hadoop.tmp.dir in system properties and HBase conf 2024-11-17T01:51:27,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/hadoop.log.dir in system properties and HBase conf 2024-11-17T01:51:27,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-17T01:51:27,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-17T01:51:27,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-17T01:51:27,365 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-17T01:51:27,453 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:51:27,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-17T01:51:27,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-17T01:51:27,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-17T01:51:27,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T01:51:27,460 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-17T01:51:27,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-17T01:51:27,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T01:51:27,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T01:51:27,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-17T01:51:27,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/nfs.dump.dir in system properties and HBase conf 2024-11-17T01:51:27,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/java.io.tmpdir in system properties and HBase conf 2024-11-17T01:51:27,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T01:51:27,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-17T01:51:27,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-17T01:51:28,455 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-17T01:51:28,523 INFO [Time-limited test {}] log.Log(170): Logging initialized @2332ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-17T01:51:28,584 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T01:51:28,635 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T01:51:28,653 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T01:51:28,653 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T01:51:28,654 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-17T01:51:28,664 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T01:51:28,667 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/hadoop.log.dir/,AVAILABLE} 2024-11-17T01:51:28,667 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T01:51:28,842 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/java.io.tmpdir/jetty-localhost-34971-hadoop-hdfs-3_4_1-tests_jar-_-any-2095014411102580843/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T01:51:28,848 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:34971} 2024-11-17T01:51:28,849 INFO [Time-limited test {}] server.Server(415): Started @2659ms 2024-11-17T01:51:29,323 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T01:51:29,332 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T01:51:29,333 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T01:51:29,333 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T01:51:29,333 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-17T01:51:29,334 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/hadoop.log.dir/,AVAILABLE} 2024-11-17T01:51:29,335 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T01:51:29,448 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/java.io.tmpdir/jetty-localhost-42549-hadoop-hdfs-3_4_1-tests_jar-_-any-12464375801064589019/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T01:51:29,449 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:42549} 2024-11-17T01:51:29,449 INFO [Time-limited test {}] server.Server(415): Started @3259ms 2024-11-17T01:51:29,501 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-17T01:51:30,360 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/cluster_97028d77-d23a-8219-e606-b0c441f49b52/data/data1/current/BP-2119396547-172.17.0.2-1731808288007/current, will proceed with Du for space computation calculation, 2024-11-17T01:51:30,360 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/cluster_97028d77-d23a-8219-e606-b0c441f49b52/data/data2/current/BP-2119396547-172.17.0.2-1731808288007/current, will proceed with Du for space computation calculation, 2024-11-17T01:51:30,386 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-17T01:51:30,430 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x51d3e98082862ba6 with lease ID 0x1413c810dcb46117: Processing first storage report for DS-f6c2b931-c614-47dc-bec1-4f089ce04bd7 from datanode DatanodeRegistration(127.0.0.1:45229, datanodeUuid=6f20295a-4c5e-4b90-8d70-b1b119004aa5, infoPort=37607, infoSecurePort=0, ipcPort=36589, storageInfo=lv=-57;cid=testClusterID;nsid=11552392;c=1731808288007) 2024-11-17T01:51:30,431 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x51d3e98082862ba6 with lease ID 0x1413c810dcb46117: from storage DS-f6c2b931-c614-47dc-bec1-4f089ce04bd7 node DatanodeRegistration(127.0.0.1:45229, datanodeUuid=6f20295a-4c5e-4b90-8d70-b1b119004aa5, infoPort=37607, infoSecurePort=0, ipcPort=36589, storageInfo=lv=-57;cid=testClusterID;nsid=11552392;c=1731808288007), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-17T01:51:30,432 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x51d3e98082862ba6 with lease ID 0x1413c810dcb46117: Processing first storage report for DS-9147b595-954f-495f-84b9-d2456fe0d08e from datanode DatanodeRegistration(127.0.0.1:45229, datanodeUuid=6f20295a-4c5e-4b90-8d70-b1b119004aa5, infoPort=37607, infoSecurePort=0, ipcPort=36589, storageInfo=lv=-57;cid=testClusterID;nsid=11552392;c=1731808288007) 2024-11-17T01:51:30,432 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x51d3e98082862ba6 with lease ID 0x1413c810dcb46117: from storage DS-9147b595-954f-495f-84b9-d2456fe0d08e node DatanodeRegistration(127.0.0.1:45229, datanodeUuid=6f20295a-4c5e-4b90-8d70-b1b119004aa5, infoPort=37607, infoSecurePort=0, ipcPort=36589, storageInfo=lv=-57;cid=testClusterID;nsid=11552392;c=1731808288007), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-17T01:51:30,505 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8 2024-11-17T01:51:30,570 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/cluster_97028d77-d23a-8219-e606-b0c441f49b52/zookeeper_0, clientPort=59270, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/cluster_97028d77-d23a-8219-e606-b0c441f49b52/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/cluster_97028d77-d23a-8219-e606-b0c441f49b52/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-17T01:51:30,579 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59270 2024-11-17T01:51:30,589 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T01:51:30,591 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T01:51:30,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741825_1001 (size=7) 2024-11-17T01:51:31,202 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1 with version=8 2024-11-17T01:51:31,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/hbase-staging 2024-11-17T01:51:31,286 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-17T01:51:31,525 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c6c0d7a0a7c0:0 server-side Connection retries=45 2024-11-17T01:51:31,533 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:51:31,534 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T01:51:31,539 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T01:51:31,539 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:51:31,539 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T01:51:31,681 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-17T01:51:31,741 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-17T01:51:31,753 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-17T01:51:31,758 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T01:51:31,782 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 86739 (auto-detected) 2024-11-17T01:51:31,783 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-17T01:51:31,801 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45023 2024-11-17T01:51:31,823 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45023 connecting to ZooKeeper ensemble=127.0.0.1:59270 2024-11-17T01:51:31,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:450230x0, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T01:51:31,950 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45023-0x10147c6f9be0000 connected 2024-11-17T01:51:32,065 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T01:51:32,068 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T01:51:32,078 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T01:51:32,081 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1, hbase.cluster.distributed=false 2024-11-17T01:51:32,102 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T01:51:32,106 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45023 2024-11-17T01:51:32,107 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45023 2024-11-17T01:51:32,107 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45023 2024-11-17T01:51:32,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45023 2024-11-17T01:51:32,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45023 2024-11-17T01:51:32,206 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c6c0d7a0a7c0:0 server-side Connection retries=45 2024-11-17T01:51:32,207 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:51:32,207 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T01:51:32,208 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T01:51:32,208 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:51:32,208 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T01:51:32,211 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T01:51:32,214 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T01:51:32,215 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41157 2024-11-17T01:51:32,217 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41157 connecting to ZooKeeper ensemble=127.0.0.1:59270 2024-11-17T01:51:32,219 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T01:51:32,222 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T01:51:32,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:411570x0, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T01:51:32,256 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:411570x0, quorum=127.0.0.1:59270, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T01:51:32,256 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41157-0x10147c6f9be0001 connected 2024-11-17T01:51:32,261 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T01:51:32,270 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T01:51:32,273 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41157-0x10147c6f9be0001, quorum=127.0.0.1:59270, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T01:51:32,277 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41157-0x10147c6f9be0001, quorum=127.0.0.1:59270, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T01:51:32,278 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41157 2024-11-17T01:51:32,278 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41157 2024-11-17T01:51:32,279 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41157 2024-11-17T01:51:32,280 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41157 2024-11-17T01:51:32,280 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41157 2024-11-17T01:51:32,295 DEBUG [M:0;c6c0d7a0a7c0:45023 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c6c0d7a0a7c0:45023 2024-11-17T01:51:32,297 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c6c0d7a0a7c0,45023,1731808291372 2024-11-17T01:51:32,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41157-0x10147c6f9be0001, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T01:51:32,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T01:51:32,316 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c6c0d7a0a7c0,45023,1731808291372 2024-11-17T01:51:32,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:51:32,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41157-0x10147c6f9be0001, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T01:51:32,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41157-0x10147c6f9be0001, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:51:32,347 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-17T01:51:32,349 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c6c0d7a0a7c0,45023,1731808291372 from backup master directory 2024-11-17T01:51:32,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c6c0d7a0a7c0,45023,1731808291372 2024-11-17T01:51:32,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41157-0x10147c6f9be0001, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T01:51:32,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T01:51:32,360 WARN [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T01:51:32,360 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c6c0d7a0a7c0,45023,1731808291372 2024-11-17T01:51:32,362 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-17T01:51:32,363 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-17T01:51:32,419 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/hbase.id] with ID: 48ead001-edd0-44c1-827a-ff10abe48617 2024-11-17T01:51:32,420 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/.tmp/hbase.id 2024-11-17T01:51:32,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741826_1002 (size=42) 2024-11-17T01:51:32,836 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/.tmp/hbase.id]:[hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/hbase.id] 2024-11-17T01:51:32,884 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T01:51:32,889 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-17T01:51:32,906 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-17T01:51:32,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41157-0x10147c6f9be0001, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:51:32,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:51:32,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741827_1003 (size=196) 2024-11-17T01:51:33,388 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:51:33,391 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-17T01:51:33,404 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:33,408 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T01:51:33,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741828_1004 (size=1189) 2024-11-17T01:51:33,855 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/data/master/store 2024-11-17T01:51:33,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741829_1005 (size=34) 2024-11-17T01:51:34,280 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-17T01:51:34,283 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:51:34,284 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T01:51:34,284 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:51:34,284 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:51:34,286 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T01:51:34,286 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:51:34,286 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:51:34,287 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731808294284Disabling compacts and flushes for region at 1731808294284Disabling writes for close at 1731808294286 (+2 ms)Writing region close event to WAL at 1731808294286Closed at 1731808294286 2024-11-17T01:51:34,290 WARN [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/data/master/store/.initializing 2024-11-17T01:51:34,290 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/WALs/c6c0d7a0a7c0,45023,1731808291372 2024-11-17T01:51:34,298 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T01:51:34,313 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6c0d7a0a7c0%2C45023%2C1731808291372, suffix=, logDir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/WALs/c6c0d7a0a7c0,45023,1731808291372, archiveDir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/oldWALs, maxLogs=10 2024-11-17T01:51:34,336 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/WALs/c6c0d7a0a7c0,45023,1731808291372/c6c0d7a0a7c0%2C45023%2C1731808291372.1731808294318, exclude list is [], retry=0 2024-11-17T01:51:34,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45229,DS-f6c2b931-c614-47dc-bec1-4f089ce04bd7,DISK] 2024-11-17T01:51:34,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-17T01:51:34,397 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/WALs/c6c0d7a0a7c0,45023,1731808291372/c6c0d7a0a7c0%2C45023%2C1731808291372.1731808294318 2024-11-17T01:51:34,398 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37607:37607)] 2024-11-17T01:51:34,398 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:51:34,399 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:51:34,402 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:51:34,403 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:51:34,437 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:51:34,461 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-17T01:51:34,464 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:51:34,467 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:51:34,467 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:51:34,471 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-17T01:51:34,471 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:51:34,472 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:51:34,472 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:51:34,475 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-17T01:51:34,476 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:51:34,477 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:51:34,477 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:51:34,481 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-17T01:51:34,481 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:51:34,482 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:51:34,483 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:51:34,486 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:51:34,488 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:51:34,493 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:51:34,494 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:51:34,497 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T01:51:34,501 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:51:34,505 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:51:34,507 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67711725, jitterRate=0.008983328938484192}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T01:51:34,513 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731808294414Initializing all the Stores at 1731808294416 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808294417 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808294417Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808294418 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808294418Cleaning up temporary data from old regions at 1731808294494 (+76 ms)Region opened successfully at 1731808294513 (+19 ms) 2024-11-17T01:51:34,514 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-17T01:51:34,547 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f654f69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c6c0d7a0a7c0/172.17.0.2:0 2024-11-17T01:51:34,585 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-17T01:51:34,594 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-17T01:51:34,594 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-17T01:51:34,597 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T01:51:34,598 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-17T01:51:34,603 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-17T01:51:34,603 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-17T01:51:34,627 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-17T01:51:34,635 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-17T01:51:34,692 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-17T01:51:34,695 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-17T01:51:34,696 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-17T01:51:34,704 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-17T01:51:34,706 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-17T01:51:34,710 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-17T01:51:34,721 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-17T01:51:34,722 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-17T01:51:34,733 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-17T01:51:34,749 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-17T01:51:34,758 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-17T01:51:34,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T01:51:34,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41157-0x10147c6f9be0001, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T01:51:34,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:51:34,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41157-0x10147c6f9be0001, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:51:34,774 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c6c0d7a0a7c0,45023,1731808291372, sessionid=0x10147c6f9be0000, setting cluster-up flag (Was=false) 2024-11-17T01:51:34,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41157-0x10147c6f9be0001, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:51:34,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:51:34,829 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-17T01:51:34,832 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c6c0d7a0a7c0,45023,1731808291372 2024-11-17T01:51:34,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:51:34,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41157-0x10147c6f9be0001, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:51:34,875 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-17T01:51:34,877 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c6c0d7a0a7c0,45023,1731808291372 2024-11-17T01:51:34,883 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-17T01:51:34,886 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(746): ClusterId : 48ead001-edd0-44c1-827a-ff10abe48617 2024-11-17T01:51:34,888 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T01:51:34,903 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T01:51:34,903 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T01:51:34,914 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T01:51:34,915 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b17866, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c6c0d7a0a7c0/172.17.0.2:0 2024-11-17T01:51:34,928 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c6c0d7a0a7c0:41157 2024-11-17T01:51:34,930 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T01:51:34,931 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T01:51:34,931 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T01:51:34,933 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(2659): reportForDuty to master=c6c0d7a0a7c0,45023,1731808291372 with port=41157, startcode=1731808292173 2024-11-17T01:51:34,943 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T01:51:34,958 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T01:51:34,967 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-17T01:51:34,973 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-17T01:51:34,976 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47157, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T01:51:34,982 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45023 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:34,979 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c6c0d7a0a7c0,45023,1731808291372 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-17T01:51:34,988 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c6c0d7a0a7c0:0, corePoolSize=5, maxPoolSize=5 2024-11-17T01:51:34,988 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c6c0d7a0a7c0:0, corePoolSize=5, maxPoolSize=5 2024-11-17T01:51:34,988 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=5, maxPoolSize=5 2024-11-17T01:51:34,989 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=5, maxPoolSize=5 2024-11-17T01:51:34,989 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c6c0d7a0a7c0:0, corePoolSize=10, maxPoolSize=10 2024-11-17T01:51:34,989 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:51:34,989 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=2, maxPoolSize=2 2024-11-17T01:51:34,989 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:51:34,991 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731808324991 2024-11-17T01:51:34,993 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-17T01:51:34,995 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-17T01:51:34,995 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T01:51:34,996 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-17T01:51:34,999 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-17T01:51:35,000 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-17T01:51:35,000 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-17T01:51:35,000 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-17T01:51:35,002 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:51:35,002 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-17T01:51:35,001 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:35,005 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-17T01:51:35,007 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-17T01:51:35,007 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-17T01:51:35,011 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-17T01:51:35,012 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-17T01:51:35,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741831_1007 (size=1321) 2024-11-17T01:51:35,013 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-17T01:51:35,013 WARN [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-17T01:51:35,014 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.large.0-1731808295013,5,FailOnTimeoutGroup] 2024-11-17T01:51:35,014 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.small.0-1731808295014,5,FailOnTimeoutGroup] 2024-11-17T01:51:35,014 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:35,014 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-17T01:51:35,016 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:35,016 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:35,114 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(2659): reportForDuty to master=c6c0d7a0a7c0,45023,1731808291372 with port=41157, startcode=1731808292173 2024-11-17T01:51:35,116 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45023 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:35,118 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45023 {}] master.ServerManager(517): Registering regionserver=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:35,126 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1 2024-11-17T01:51:35,126 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35241 2024-11-17T01:51:35,126 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T01:51:35,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T01:51:35,155 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] zookeeper.ZKUtil(111): regionserver:41157-0x10147c6f9be0001, quorum=127.0.0.1:59270, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:35,155 WARN [RS:0;c6c0d7a0a7c0:41157 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T01:51:35,156 INFO [RS:0;c6c0d7a0a7c0:41157 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T01:51:35,156 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/WALs/c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:35,158 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c6c0d7a0a7c0,41157,1731808292173] 2024-11-17T01:51:35,178 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T01:51:35,193 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T01:51:35,197 INFO [RS:0;c6c0d7a0a7c0:41157 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T01:51:35,198 INFO [RS:0;c6c0d7a0a7c0:41157 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:35,199 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T01:51:35,204 INFO [RS:0;c6c0d7a0a7c0:41157 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T01:51:35,205 INFO [RS:0;c6c0d7a0a7c0:41157 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:35,205 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:51:35,205 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:51:35,206 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:51:35,206 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:51:35,206 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:51:35,206 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0, corePoolSize=2, maxPoolSize=2 2024-11-17T01:51:35,206 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:51:35,206 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:51:35,206 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:51:35,207 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:51:35,207 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:51:35,207 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:51:35,207 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c6c0d7a0a7c0:0, corePoolSize=3, maxPoolSize=3 2024-11-17T01:51:35,207 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0, corePoolSize=3, maxPoolSize=3 2024-11-17T01:51:35,208 INFO [RS:0;c6c0d7a0a7c0:41157 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:35,208 INFO [RS:0;c6c0d7a0a7c0:41157 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:35,209 INFO [RS:0;c6c0d7a0a7c0:41157 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:35,209 INFO [RS:0;c6c0d7a0a7c0:41157 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:35,209 INFO [RS:0;c6c0d7a0a7c0:41157 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:35,209 INFO [RS:0;c6c0d7a0a7c0:41157 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,41157,1731808292173-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T01:51:35,228 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T01:51:35,230 INFO [RS:0;c6c0d7a0a7c0:41157 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,41157,1731808292173-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:35,231 INFO [RS:0;c6c0d7a0a7c0:41157 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:35,231 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.Replication(171): c6c0d7a0a7c0,41157,1731808292173 started 2024-11-17T01:51:35,248 INFO [RS:0;c6c0d7a0a7c0:41157 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:35,249 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(1482): Serving as c6c0d7a0a7c0,41157,1731808292173, RpcServer on c6c0d7a0a7c0/172.17.0.2:41157, sessionid=0x10147c6f9be0001 2024-11-17T01:51:35,249 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T01:51:35,249 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:35,250 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6c0d7a0a7c0,41157,1731808292173' 2024-11-17T01:51:35,250 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T01:51:35,251 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T01:51:35,251 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T01:51:35,251 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T01:51:35,252 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:35,252 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6c0d7a0a7c0,41157,1731808292173' 2024-11-17T01:51:35,252 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T01:51:35,252 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T01:51:35,253 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T01:51:35,253 INFO [RS:0;c6c0d7a0a7c0:41157 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T01:51:35,253 INFO [RS:0;c6c0d7a0a7c0:41157 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T01:51:35,360 INFO [RS:0;c6c0d7a0a7c0:41157 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T01:51:35,363 INFO [RS:0;c6c0d7a0a7c0:41157 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6c0d7a0a7c0%2C41157%2C1731808292173, suffix=, logDir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/WALs/c6c0d7a0a7c0,41157,1731808292173, archiveDir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/oldWALs, maxLogs=32 2024-11-17T01:51:35,379 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/WALs/c6c0d7a0a7c0,41157,1731808292173/c6c0d7a0a7c0%2C41157%2C1731808292173.1731808295366, exclude list is [], retry=0 2024-11-17T01:51:35,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45229,DS-f6c2b931-c614-47dc-bec1-4f089ce04bd7,DISK] 2024-11-17T01:51:35,386 INFO [RS:0;c6c0d7a0a7c0:41157 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/WALs/c6c0d7a0a7c0,41157,1731808292173/c6c0d7a0a7c0%2C41157%2C1731808292173.1731808295366 2024-11-17T01:51:35,387 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37607:37607)] 2024-11-17T01:51:35,415 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-17T01:51:35,416 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1 2024-11-17T01:51:35,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741833_1009 (size=32) 2024-11-17T01:51:35,830 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:51:35,836 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T01:51:35,840 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T01:51:35,840 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:51:35,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:51:35,842 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T01:51:35,845 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T01:51:35,846 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:51:35,847 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:51:35,848 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T01:51:35,851 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T01:51:35,851 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:51:35,852 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:51:35,853 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T01:51:35,856 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T01:51:35,856 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:51:35,857 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:51:35,858 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T01:51:35,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740 2024-11-17T01:51:35,861 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740 2024-11-17T01:51:35,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T01:51:35,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T01:51:35,869 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T01:51:35,873 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T01:51:35,883 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:51:35,885 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62114524, jitterRate=-0.0744214653968811}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T01:51:35,888 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731808295831Initializing all the Stores at 1731808295833 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808295833Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808295835 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808295835Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808295835Cleaning up temporary data from old regions at 1731808295868 (+33 ms)Region opened successfully at 1731808295888 (+20 ms) 2024-11-17T01:51:35,888 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T01:51:35,889 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T01:51:35,889 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T01:51:35,889 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T01:51:35,889 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T01:51:35,891 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T01:51:35,891 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731808295888Disabling compacts and flushes for region at 1731808295888Disabling writes for close at 1731808295889 (+1 ms)Writing region close event to WAL at 1731808295890 (+1 ms)Closed at 1731808295891 (+1 ms) 2024-11-17T01:51:35,895 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T01:51:35,895 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-17T01:51:35,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-17T01:51:35,917 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T01:51:35,921 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-17T01:51:36,074 DEBUG [c6c0d7a0a7c0:45023 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-17T01:51:36,085 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:36,092 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,41157,1731808292173, state=OPENING 2024-11-17T01:51:36,117 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-17T01:51:36,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41157-0x10147c6f9be0001, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:51:36,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:51:36,126 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:51:36,126 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:51:36,128 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T01:51:36,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,41157,1731808292173}] 2024-11-17T01:51:36,307 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T01:51:36,310 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46785, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T01:51:36,321 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-17T01:51:36,321 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T01:51:36,322 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-17T01:51:36,326 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6c0d7a0a7c0%2C41157%2C1731808292173.meta, suffix=.meta, logDir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/WALs/c6c0d7a0a7c0,41157,1731808292173, archiveDir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/oldWALs, maxLogs=32 2024-11-17T01:51:36,343 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/WALs/c6c0d7a0a7c0,41157,1731808292173/c6c0d7a0a7c0%2C41157%2C1731808292173.meta.1731808296328.meta, exclude list is [], retry=0 2024-11-17T01:51:36,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45229,DS-f6c2b931-c614-47dc-bec1-4f089ce04bd7,DISK] 2024-11-17T01:51:36,349 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/WALs/c6c0d7a0a7c0,41157,1731808292173/c6c0d7a0a7c0%2C41157%2C1731808292173.meta.1731808296328.meta 2024-11-17T01:51:36,350 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37607:37607)] 2024-11-17T01:51:36,350 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:51:36,352 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-17T01:51:36,354 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-17T01:51:36,360 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-17T01:51:36,365 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-17T01:51:36,366 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:51:36,366 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-17T01:51:36,366 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-17T01:51:36,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T01:51:36,371 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T01:51:36,371 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:51:36,372 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:51:36,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T01:51:36,374 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T01:51:36,374 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:51:36,375 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:51:36,375 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T01:51:36,377 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T01:51:36,377 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:51:36,378 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:51:36,378 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T01:51:36,380 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T01:51:36,380 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:51:36,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:51:36,381 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T01:51:36,382 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740 2024-11-17T01:51:36,385 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740 2024-11-17T01:51:36,388 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T01:51:36,388 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T01:51:36,389 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T01:51:36,393 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T01:51:36,395 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69612468, jitterRate=0.03730660676956177}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T01:51:36,395 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-17T01:51:36,397 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731808296367Writing region info on filesystem at 1731808296367Initializing all the Stores at 1731808296369 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808296369Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808296369Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808296369Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808296369Cleaning up temporary data from old regions at 1731808296388 (+19 ms)Running coprocessor post-open hooks at 1731808296395 (+7 ms)Region opened successfully at 1731808296397 (+2 ms) 2024-11-17T01:51:36,405 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731808296298 2024-11-17T01:51:36,419 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-17T01:51:36,419 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-17T01:51:36,421 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:36,424 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,41157,1731808292173, state=OPEN 2024-11-17T01:51:36,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41157-0x10147c6f9be0001, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T01:51:36,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T01:51:36,525 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:51:36,526 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:51:36,526 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:36,531 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-17T01:51:36,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,41157,1731808292173 in 396 msec 2024-11-17T01:51:36,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-17T01:51:36,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 630 msec 2024-11-17T01:51:36,542 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T01:51:36,543 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-17T01:51:36,563 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:51:36,565 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:51:36,585 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:51:36,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54885, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:51:36,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6940 sec 2024-11-17T01:51:36,609 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731808296609, completionTime=-1 2024-11-17T01:51:36,612 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-17T01:51:36,613 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T01:51:36,667 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-17T01:51:36,667 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731808356667 2024-11-17T01:51:36,667 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731808416667 2024-11-17T01:51:36,667 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 54 msec 2024-11-17T01:51:36,670 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,45023,1731808291372-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:36,670 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,45023,1731808291372-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:36,670 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,45023,1731808291372-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:36,681 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c6c0d7a0a7c0:45023, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:36,681 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:36,682 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:36,690 DEBUG [master/c6c0d7a0a7c0:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-17T01:51:36,714 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.354sec 2024-11-17T01:51:36,716 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-17T01:51:36,717 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-17T01:51:36,719 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-17T01:51:36,719 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-17T01:51:36,719 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-17T01:51:36,720 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,45023,1731808291372-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T01:51:36,721 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,45023,1731808291372-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-17T01:51:36,729 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-17T01:51:36,730 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-17T01:51:36,730 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,45023,1731808291372-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:51:36,798 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:36,800 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-17T01:51:36,800 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-17T01:51:36,803 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:51:36,806 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:51:36,815 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:51:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:51:36,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:51:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24393d00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:51:36,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:51:36,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:51:36,826 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:51:36,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:36,830 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:51:36,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:51:36,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:51:36,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:51:36,863 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c6c0d7a0a7c0,45023,1731808291372 2024-11-17T01:51:36,878 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=224, ProcessCount=11, AvailableMemoryMB=5572 2024-11-17T01:51:36,888 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T01:51:36,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c6c0d7a0a7c0,45023,1731808291372 2024-11-17T01:51:36,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b65baa9 2024-11-17T01:51:36,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T01:51:36,899 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T01:51:36,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T01:51:36,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:51:36,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T01:51:36,918 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T01:51:36,920 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-17T01:51:36,920 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:51:36,924 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T01:51:36,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T01:51:36,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741835_1011 (size=1061) 2024-11-17T01:51:37,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T01:51:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T01:51:37,343 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1 2024-11-17T01:51:37,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741836_1012 (size=53) 2024-11-17T01:51:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T01:51:37,755 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:51:37,755 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5d8a196529c6f40b7321f17bb79ff0a8, disabling compactions & flushes 2024-11-17T01:51:37,755 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:37,755 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:37,756 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. after waiting 0 ms 2024-11-17T01:51:37,756 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:37,756 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:37,756 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5d8a196529c6f40b7321f17bb79ff0a8: Waiting for close lock at 1731808297755Disabling compacts and flushes for region at 1731808297755Disabling writes for close at 1731808297756 (+1 ms)Writing region close event to WAL at 1731808297756Closed at 1731808297756 2024-11-17T01:51:37,759 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T01:51:37,763 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731808297759"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731808297759"}]},"ts":"1731808297759"} 2024-11-17T01:51:37,768 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T01:51:37,771 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T01:51:37,773 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808297771"}]},"ts":"1731808297771"} 2024-11-17T01:51:37,779 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T01:51:37,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d8a196529c6f40b7321f17bb79ff0a8, ASSIGN}] 2024-11-17T01:51:37,784 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d8a196529c6f40b7321f17bb79ff0a8, ASSIGN 2024-11-17T01:51:37,785 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d8a196529c6f40b7321f17bb79ff0a8, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,41157,1731808292173; forceNewPlan=false, retain=false 2024-11-17T01:51:37,937 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5d8a196529c6f40b7321f17bb79ff0a8, regionState=OPENING, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:37,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d8a196529c6f40b7321f17bb79ff0a8, ASSIGN because future has completed 2024-11-17T01:51:37,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173}] 2024-11-17T01:51:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T01:51:38,104 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:38,104 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:51:38,104 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:38,105 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:51:38,105 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:38,105 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:38,107 INFO [StoreOpener-5d8a196529c6f40b7321f17bb79ff0a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:38,111 INFO [StoreOpener-5d8a196529c6f40b7321f17bb79ff0a8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:51:38,111 INFO [StoreOpener-5d8a196529c6f40b7321f17bb79ff0a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d8a196529c6f40b7321f17bb79ff0a8 columnFamilyName A 2024-11-17T01:51:38,111 DEBUG [StoreOpener-5d8a196529c6f40b7321f17bb79ff0a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:51:38,113 INFO [StoreOpener-5d8a196529c6f40b7321f17bb79ff0a8-1 {}] regionserver.HStore(327): Store=5d8a196529c6f40b7321f17bb79ff0a8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:51:38,113 INFO [StoreOpener-5d8a196529c6f40b7321f17bb79ff0a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:38,115 INFO [StoreOpener-5d8a196529c6f40b7321f17bb79ff0a8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:51:38,116 INFO [StoreOpener-5d8a196529c6f40b7321f17bb79ff0a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d8a196529c6f40b7321f17bb79ff0a8 columnFamilyName B 2024-11-17T01:51:38,116 DEBUG [StoreOpener-5d8a196529c6f40b7321f17bb79ff0a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:51:38,117 INFO [StoreOpener-5d8a196529c6f40b7321f17bb79ff0a8-1 {}] regionserver.HStore(327): Store=5d8a196529c6f40b7321f17bb79ff0a8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:51:38,117 INFO [StoreOpener-5d8a196529c6f40b7321f17bb79ff0a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:38,119 INFO [StoreOpener-5d8a196529c6f40b7321f17bb79ff0a8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:51:38,120 INFO [StoreOpener-5d8a196529c6f40b7321f17bb79ff0a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d8a196529c6f40b7321f17bb79ff0a8 columnFamilyName C 2024-11-17T01:51:38,120 DEBUG [StoreOpener-5d8a196529c6f40b7321f17bb79ff0a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:51:38,121 INFO [StoreOpener-5d8a196529c6f40b7321f17bb79ff0a8-1 {}] regionserver.HStore(327): Store=5d8a196529c6f40b7321f17bb79ff0a8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:51:38,121 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:38,122 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:38,123 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:38,124 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:38,126 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:38,126 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:38,127 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T01:51:38,129 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:38,132 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:51:38,133 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5d8a196529c6f40b7321f17bb79ff0a8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71302978, jitterRate=0.06249716877937317}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T01:51:38,133 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:38,134 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5d8a196529c6f40b7321f17bb79ff0a8: Running coprocessor pre-open hook at 1731808298105Writing region info on filesystem at 1731808298105Initializing all the Stores at 1731808298107 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808298107Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808298107Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808298107Cleaning up temporary data from old regions at 1731808298126 (+19 ms)Running coprocessor post-open hooks at 1731808298133 (+7 ms)Region opened successfully at 1731808298134 (+1 ms) 2024-11-17T01:51:38,136 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., pid=6, masterSystemTime=1731808298097 2024-11-17T01:51:38,140 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:38,140 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:38,141 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5d8a196529c6f40b7321f17bb79ff0a8, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:38,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 because future has completed 2024-11-17T01:51:38,152 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-17T01:51:38,152 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 in 206 msec 2024-11-17T01:51:38,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-17T01:51:38,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d8a196529c6f40b7321f17bb79ff0a8, ASSIGN in 372 msec 2024-11-17T01:51:38,159 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T01:51:38,159 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808298159"}]},"ts":"1731808298159"} 2024-11-17T01:51:38,162 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T01:51:38,164 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T01:51:38,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2550 sec 2024-11-17T01:51:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T01:51:39,086 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:51:39,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,091 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:51:39,092 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:51:39,093 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:51:39,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:51:39,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:51:39,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ecdd502, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:51:39,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:51:39,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:51:39,097 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:51:39,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c5fed3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,099 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:51:39,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:51:39,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:51:39,104 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:51:39,106 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T01:51:39,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c6c0d7a0a7c0,45023,1731808291372 2024-11-17T01:51:39,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@23bae5e5 2024-11-17T01:51:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T01:51:39,114 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T01:51:39,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,119 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:51:39,119 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:51:39,120 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:51:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:51:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:51:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b9dd8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:51:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:51:39,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:51:39,123 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:51:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64257f01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,126 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,126 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:51:39,126 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:51:39,128 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:51:39,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:51:39,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:51:39,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a27ead7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:51:39,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:51:39,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:51:39,132 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:51:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ba4844, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,139 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:51:39,139 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:51:39,141 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:51:39,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:51:39,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:51:39,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643162ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:51:39,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:51:39,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:51:39,146 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:51:39,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1380f017, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,148 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:51:39,149 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:51:39,151 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:51:39,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:51:39,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:51:39,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6d3ad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:51:39,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:51:39,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:51:39,157 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:51:39,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763c2183, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:51:39,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:51:39,162 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:51:39,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:51:39,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:51:39,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18dcd239, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:51:39,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:51:39,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:51:39,165 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:51:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c50a81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,168 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:51:39,168 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:51:39,169 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:51:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:51:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:51:39,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af5f6a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:51:39,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:51:39,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:51:39,173 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:51:39,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cb8306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,175 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:51:39,175 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:51:39,176 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:51:39,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:51:39,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:51:39,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@276c4225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:51:39,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:51:39,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:51:39,179 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:51:39,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f23297c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,181 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:51:39,181 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:51:39,182 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:51:39,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:51:39,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:51:39,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d0e529e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:51:39,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:51:39,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:51:39,185 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:51:39,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aaa86d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,187 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:51:39,187 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:51:39,188 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:51:39,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:51:39,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:51:39,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d25ecbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:51:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:51:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:51:39,191 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:51:39,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@315b49bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:51:39,197 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:51:39,197 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:51:39,198 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:51:39,198 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:51:39,198 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:51:39,198 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:51:39,199 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:51:39,197 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:51:39,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:51:39,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:51:39,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:51:39,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:51:39,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:51:39,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:51:39,199 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:51:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:51:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:51:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:51:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:51:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:51:39,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:51:39,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:51:39,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:51:39,202 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:51:39,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:51:39,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:51:39,203 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:51:39,203 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:51:39,203 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:51:39,203 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:51:39,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:51:39,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:51:39,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:51:39,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:51:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:51:39,206 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:51:39,209 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:51:39,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:51:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:51:39,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:51:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:51:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:51:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:51:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:51:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:51:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-17T01:51:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T01:51:39,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:51:39,226 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:51:39,228 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:51:39,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:51:39,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T01:51:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:39,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:39,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:39,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:39,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:39,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:39,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T01:51:39,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T01:51:39,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:39,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:39,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:39,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808359418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:39,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:39,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:39,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808359420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:39,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808359432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:39,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/4245f6acd1114582b93c091b842c9d93 is 50, key is test_row_0/A:col10/1731808299299/Put/seqid=0 2024-11-17T01:51:39,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:39,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808299410, max timestamp=1731808299431Num uniques 859; 2024-11-17T01:51:39,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:51:39,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808299410, max timestamp=1731808299431Num uniques 859; 2024-11-17T01:51:39,490 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:51:39,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808299410, max timestamp=1731808299431Num uniques 859; 2024-11-17T01:51:39,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:51:39,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741837_1013 (size=14341) 2024-11-17T01:51:39,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/4245f6acd1114582b93c091b842c9d93 2024-11-17T01:51:39,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T01:51:39,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:39,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808359586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:39,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T01:51:39,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:39,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:39,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808359590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:39,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:39,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:39,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808359596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:39,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:39,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:39,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:39,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:39,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:39,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:39,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808359614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:39,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:39,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808359620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:39,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:39,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:39,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:39,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/6281bf62b1e84a67b688052abf7ce33a is 50, key is test_row_0/B:col10/1731808299299/Put/seqid=0 2024-11-17T01:51:39,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741838_1014 (size=12001) 2024-11-17T01:51:39,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/6281bf62b1e84a67b688052abf7ce33a 2024-11-17T01:51:39,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808359724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:39,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:39,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:39,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808359725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:39,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:39,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808359736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:39,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:39,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808359737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:39,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:39,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808359745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:39,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:39,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:39,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/646e2588863d41888035ce3ecaf4423c is 50, key is test_row_0/C:col10/1731808299299/Put/seqid=0 2024-11-17T01:51:39,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T01:51:39,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:39,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741839_1015 (size=12001) 2024-11-17T01:51:39,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:39,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:39,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:39,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:39,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/646e2588863d41888035ce3ecaf4423c 2024-11-17T01:51:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:39,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/4245f6acd1114582b93c091b842c9d93 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/4245f6acd1114582b93c091b842c9d93 2024-11-17T01:51:39,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/4245f6acd1114582b93c091b842c9d93, entries=200, sequenceid=13, filesize=14.0 K 2024-11-17T01:51:39,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/6281bf62b1e84a67b688052abf7ce33a as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/6281bf62b1e84a67b688052abf7ce33a 2024-11-17T01:51:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T01:51:39,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/6281bf62b1e84a67b688052abf7ce33a, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T01:51:39,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/646e2588863d41888035ce3ecaf4423c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/646e2588863d41888035ce3ecaf4423c 2024-11-17T01:51:39,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/646e2588863d41888035ce3ecaf4423c, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T01:51:39,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 5d8a196529c6f40b7321f17bb79ff0a8 in 602ms, sequenceid=13, compaction requested=false 2024-11-17T01:51:39,915 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T01:51:39,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:39,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-17T01:51:39,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:39,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:51:39,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:39,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:51:39,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:39,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:51:39,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T01:51:39,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:39,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:39,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:39,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:39,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:39,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/3d4c69b722e545f3819de3de50ee34a7 is 50, key is test_row_0/A:col10/1731808299594/Put/seqid=0 2024-11-17T01:51:40,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808359980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:40,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:40,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808359984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:40,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:40,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808359989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:40,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741840_1016 (size=14341) 2024-11-17T01:51:40,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808359994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:40,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808360003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:40,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T01:51:40,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:40,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:40,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:40,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:40,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:40,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808360117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:40,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:40,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808360117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:40,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808360125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:40,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:40,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808360126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:40,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:40,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808360128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:40,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:40,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T01:51:40,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:40,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:40,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:40,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:40,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:40,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:40,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808360336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:40,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808360336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:40,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808360336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:40,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:40,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808360345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:40,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:40,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808360346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:40,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T01:51:40,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/3d4c69b722e545f3819de3de50ee34a7 2024-11-17T01:51:40,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T01:51:40,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:40,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:40,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:40,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:40,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:40,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:40,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/3adc00a0b0f84b7bb6aa1a7272c90e18 is 50, key is test_row_0/B:col10/1731808299594/Put/seqid=0 2024-11-17T01:51:40,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741841_1017 (size=12001) 2024-11-17T01:51:40,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/3adc00a0b0f84b7bb6aa1a7272c90e18 2024-11-17T01:51:40,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/f5d86bf0bbf341448e43f0e4698e15da is 50, key is test_row_0/C:col10/1731808299594/Put/seqid=0 2024-11-17T01:51:40,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741842_1018 (size=12001) 2024-11-17T01:51:40,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T01:51:40,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:40,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:40,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:40,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:40,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:40,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:40,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:40,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808360655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:40,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:40,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:40,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808360656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:40,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808360656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:40,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:40,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808360665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:40,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:40,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808360665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:40,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T01:51:40,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:40,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:40,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:40,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:40,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:40,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T01:51:40,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:40,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:40,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:40,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:40,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:40,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/f5d86bf0bbf341448e43f0e4698e15da 2024-11-17T01:51:41,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/3d4c69b722e545f3819de3de50ee34a7 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/3d4c69b722e545f3819de3de50ee34a7 2024-11-17T01:51:41,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/3d4c69b722e545f3819de3de50ee34a7, entries=200, sequenceid=49, filesize=14.0 K 2024-11-17T01:51:41,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/3adc00a0b0f84b7bb6aa1a7272c90e18 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/3adc00a0b0f84b7bb6aa1a7272c90e18 2024-11-17T01:51:41,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/3adc00a0b0f84b7bb6aa1a7272c90e18, entries=150, sequenceid=49, filesize=11.7 K 2024-11-17T01:51:41,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/f5d86bf0bbf341448e43f0e4698e15da as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/f5d86bf0bbf341448e43f0e4698e15da 2024-11-17T01:51:41,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T01:51:41,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:41,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:41,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:41,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:41,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:41,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:41,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/f5d86bf0bbf341448e43f0e4698e15da, entries=150, sequenceid=49, filesize=11.7 K 2024-11-17T01:51:41,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 5d8a196529c6f40b7321f17bb79ff0a8 in 1152ms, sequenceid=49, compaction requested=false 2024-11-17T01:51:41,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:41,173 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-17T01:51:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:41,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T01:51:41,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:41,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:41,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:41,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:41,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:41,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:41,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/b4db29f6534d43c5a44b79f48494d8c8 is 50, key is test_row_0/A:col10/1731808299989/Put/seqid=0 2024-11-17T01:51:41,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T01:51:41,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:41,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:41,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:41,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:41,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:41,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:41,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741843_1019 (size=12001) 2024-11-17T01:51:41,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808361260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808361261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:41,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:41,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:41,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808361264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:41,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:41,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808361270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:41,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:41,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:41,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808361274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:41,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T01:51:41,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T01:51:41,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:41,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:41,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:41,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:41,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:41,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808361385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:41,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808361386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:41,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:41,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:41,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808361395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:41,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808361395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:41,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808361395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:41,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T01:51:41,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:41,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:41,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:41,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:41,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:41,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:41,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808361594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:41,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:41,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:41,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808361605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:41,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:41,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808361615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:41,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:41,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808361616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:41,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808361617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:41,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/b4db29f6534d43c5a44b79f48494d8c8 2024-11-17T01:51:41,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/c5bf1fee5ef84ad39421bc71b87d7071 is 50, key is test_row_0/B:col10/1731808299989/Put/seqid=0 2024-11-17T01:51:41,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T01:51:41,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:41,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:41,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:41,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:41,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:41,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741844_1020 (size=12001) 2024-11-17T01:51:41,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/c5bf1fee5ef84ad39421bc71b87d7071 2024-11-17T01:51:41,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/e613a3e7451d4a2a83765583fc949478 is 50, key is test_row_0/C:col10/1731808299989/Put/seqid=0 2024-11-17T01:51:41,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741845_1021 (size=12001) 2024-11-17T01:51:41,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/e613a3e7451d4a2a83765583fc949478 2024-11-17T01:51:41,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/b4db29f6534d43c5a44b79f48494d8c8 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/b4db29f6534d43c5a44b79f48494d8c8 2024-11-17T01:51:41,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/b4db29f6534d43c5a44b79f48494d8c8, entries=150, sequenceid=62, filesize=11.7 K 2024-11-17T01:51:41,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/c5bf1fee5ef84ad39421bc71b87d7071 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/c5bf1fee5ef84ad39421bc71b87d7071 2024-11-17T01:51:41,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/c5bf1fee5ef84ad39421bc71b87d7071, entries=150, sequenceid=62, filesize=11.7 K 2024-11-17T01:51:41,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/e613a3e7451d4a2a83765583fc949478 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/e613a3e7451d4a2a83765583fc949478 2024-11-17T01:51:41,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/e613a3e7451d4a2a83765583fc949478, entries=150, sequenceid=62, filesize=11.7 K 2024-11-17T01:51:41,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5d8a196529c6f40b7321f17bb79ff0a8 in 658ms, sequenceid=62, compaction requested=true 2024-11-17T01:51:41,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:41,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:51:41,844 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:51:41,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:41,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:51:41,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:41,844 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:51:41,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:51:41,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:51:41,849 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:51:41,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T01:51:41,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:41,850 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T01:51:41,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:41,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:41,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:41,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:41,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:41,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:41,852 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:51:41,852 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/A is initiating minor compaction (all files) 2024-11-17T01:51:41,853 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/A in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:41,853 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/4245f6acd1114582b93c091b842c9d93, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/3d4c69b722e545f3819de3de50ee34a7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/b4db29f6534d43c5a44b79f48494d8c8] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=39.7 K 2024-11-17T01:51:41,857 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4245f6acd1114582b93c091b842c9d93, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731808299265 2024-11-17T01:51:41,859 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d4c69b722e545f3819de3de50ee34a7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731808299571 2024-11-17T01:51:41,860 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4db29f6534d43c5a44b79f48494d8c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731808299989 2024-11-17T01:51:41,861 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/B is initiating minor compaction (all files) 2024-11-17T01:51:41,861 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/B in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:41,862 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/6281bf62b1e84a67b688052abf7ce33a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/3adc00a0b0f84b7bb6aa1a7272c90e18, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/c5bf1fee5ef84ad39421bc71b87d7071] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=35.2 K 2024-11-17T01:51:41,863 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 6281bf62b1e84a67b688052abf7ce33a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731808299265 2024-11-17T01:51:41,864 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 3adc00a0b0f84b7bb6aa1a7272c90e18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731808299571 2024-11-17T01:51:41,866 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting c5bf1fee5ef84ad39421bc71b87d7071, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731808299989 2024-11-17T01:51:41,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/aba3b24898ca40e7bc8bbfbfbe1e956e is 50, key is test_row_0/A:col10/1731808301270/Put/seqid=0 2024-11-17T01:51:41,887 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T01:51:41,920 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#B#compaction#10 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:41,921 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/b308fdbfb044454e836770dad70478a2 is 50, key is test_row_0/B:col10/1731808299989/Put/seqid=0 2024-11-17T01:51:41,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:41,935 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#A#compaction#11 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:41,936 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/2db4017dffcf40cfaebe5d24b4f9ac73 is 50, key is test_row_0/A:col10/1731808299989/Put/seqid=0 2024-11-17T01:51:41,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741846_1022 (size=12001) 2024-11-17T01:51:41,950 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/aba3b24898ca40e7bc8bbfbfbe1e956e 2024-11-17T01:51:41,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741848_1024 (size=12104) 2024-11-17T01:51:41,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741847_1023 (size=12104) 2024-11-17T01:51:41,991 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-17T01:51:41,992 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-17T01:51:41,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/53a3dbf3769d425a93eeac7019195f5f is 50, key is test_row_0/B:col10/1731808301270/Put/seqid=0 2024-11-17T01:51:41,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:41,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808361974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808361970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:41,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:41,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:41,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,007 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/2db4017dffcf40cfaebe5d24b4f9ac73 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/2db4017dffcf40cfaebe5d24b4f9ac73 2024-11-17T01:51:42,007 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/b308fdbfb044454e836770dad70478a2 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/b308fdbfb044454e836770dad70478a2 2024-11-17T01:51:42,008 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T01:51:42,008 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-17T01:51:42,008 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-17T01:51:42,008 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-17T01:51:42,011 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T01:51:42,011 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-17T01:51:42,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808361999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808361999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808362000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741849_1025 (size=12001) 2024-11-17T01:51:42,038 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/53a3dbf3769d425a93eeac7019195f5f 2024-11-17T01:51:42,040 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/B of 5d8a196529c6f40b7321f17bb79ff0a8 into b308fdbfb044454e836770dad70478a2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:42,040 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:42,042 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/A of 5d8a196529c6f40b7321f17bb79ff0a8 into 2db4017dffcf40cfaebe5d24b4f9ac73(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:42,042 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:42,045 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/A, priority=13, startTime=1731808301839; duration=0sec 2024-11-17T01:51:42,045 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/B, priority=13, startTime=1731808301844; duration=0sec 2024-11-17T01:51:42,046 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:51:42,046 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:A 2024-11-17T01:51:42,047 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:42,047 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:B 2024-11-17T01:51:42,047 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:51:42,052 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:51:42,053 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/C is initiating minor compaction (all files) 2024-11-17T01:51:42,053 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/C in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:42,053 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/646e2588863d41888035ce3ecaf4423c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/f5d86bf0bbf341448e43f0e4698e15da, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/e613a3e7451d4a2a83765583fc949478] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=35.2 K 2024-11-17T01:51:42,055 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 646e2588863d41888035ce3ecaf4423c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731808299265 2024-11-17T01:51:42,057 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5d86bf0bbf341448e43f0e4698e15da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731808299571 2024-11-17T01:51:42,058 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting e613a3e7451d4a2a83765583fc949478, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731808299989 2024-11-17T01:51:42,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/85e7010e894d409dbea9ca4df13b4148 is 50, key is test_row_0/C:col10/1731808301270/Put/seqid=0 2024-11-17T01:51:42,090 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#C#compaction#14 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:42,091 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/02d397dd1904478fac6b69acbc3c40cc is 50, key is test_row_0/C:col10/1731808299989/Put/seqid=0 2024-11-17T01:51:42,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741850_1026 (size=12001) 2024-11-17T01:51:42,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808362105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808362105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741851_1027 (size=12104) 2024-11-17T01:51:42,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808362125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808362125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808362125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,141 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/02d397dd1904478fac6b69acbc3c40cc as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/02d397dd1904478fac6b69acbc3c40cc 2024-11-17T01:51:42,165 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/C of 5d8a196529c6f40b7321f17bb79ff0a8 into 02d397dd1904478fac6b69acbc3c40cc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:42,165 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:42,165 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/C, priority=13, startTime=1731808301844; duration=0sec 2024-11-17T01:51:42,165 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:42,166 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:C 2024-11-17T01:51:42,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808362314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808362314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808362335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808362335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808362335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,509 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/85e7010e894d409dbea9ca4df13b4148 2024-11-17T01:51:42,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/aba3b24898ca40e7bc8bbfbfbe1e956e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/aba3b24898ca40e7bc8bbfbfbe1e956e 2024-11-17T01:51:42,534 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/aba3b24898ca40e7bc8bbfbfbe1e956e, entries=150, sequenceid=81, filesize=11.7 K 2024-11-17T01:51:42,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/53a3dbf3769d425a93eeac7019195f5f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/53a3dbf3769d425a93eeac7019195f5f 2024-11-17T01:51:42,551 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/53a3dbf3769d425a93eeac7019195f5f, entries=150, sequenceid=81, filesize=11.7 K 2024-11-17T01:51:42,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/85e7010e894d409dbea9ca4df13b4148 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/85e7010e894d409dbea9ca4df13b4148 2024-11-17T01:51:42,572 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/85e7010e894d409dbea9ca4df13b4148, entries=150, sequenceid=81, filesize=11.7 K 2024-11-17T01:51:42,577 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=80.51 KB/82440 for 5d8a196529c6f40b7321f17bb79ff0a8 in 727ms, sequenceid=81, compaction requested=false 2024-11-17T01:51:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-17T01:51:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-17T01:51:42,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-17T01:51:42,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3500 sec 2024-11-17T01:51:42,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.3740 sec 2024-11-17T01:51:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:42,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T01:51:42,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:42,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:42,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:42,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:42,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:42,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:42,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/9496a12c522c44b9941e1a47ac0b56fd is 50, key is test_row_0/A:col10/1731808301965/Put/seqid=0 2024-11-17T01:51:42,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741852_1028 (size=12001) 2024-11-17T01:51:42,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/9496a12c522c44b9941e1a47ac0b56fd 2024-11-17T01:51:42,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808362681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808362677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808362685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808362689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808362691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/d530213fbf794de494bdf1a12dc60e84 is 50, key is test_row_0/B:col10/1731808301965/Put/seqid=0 2024-11-17T01:51:42,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741853_1029 (size=12001) 2024-11-17T01:51:42,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/d530213fbf794de494bdf1a12dc60e84 2024-11-17T01:51:42,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/e5aff052a5f742c284d8fdac77b6ef07 is 50, key is test_row_0/C:col10/1731808301965/Put/seqid=0 2024-11-17T01:51:42,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808362794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808362795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808362805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808362806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808362806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:42,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741854_1030 (size=12001) 2024-11-17T01:51:42,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/e5aff052a5f742c284d8fdac77b6ef07 2024-11-17T01:51:42,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/9496a12c522c44b9941e1a47ac0b56fd as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/9496a12c522c44b9941e1a47ac0b56fd 2024-11-17T01:51:42,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/9496a12c522c44b9941e1a47ac0b56fd, entries=150, sequenceid=100, filesize=11.7 K 2024-11-17T01:51:42,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/d530213fbf794de494bdf1a12dc60e84 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/d530213fbf794de494bdf1a12dc60e84 2024-11-17T01:51:42,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/d530213fbf794de494bdf1a12dc60e84, entries=150, sequenceid=100, filesize=11.7 K 2024-11-17T01:51:42,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/e5aff052a5f742c284d8fdac77b6ef07 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/e5aff052a5f742c284d8fdac77b6ef07 2024-11-17T01:51:42,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/e5aff052a5f742c284d8fdac77b6ef07, entries=150, sequenceid=100, filesize=11.7 K 2024-11-17T01:51:42,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5d8a196529c6f40b7321f17bb79ff0a8 in 259ms, sequenceid=100, compaction requested=true 2024-11-17T01:51:42,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:42,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:51:42,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:42,895 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:51:42,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:51:42,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:42,895 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:51:42,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:51:42,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:51:42,898 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:51:42,898 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:51:42,898 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/B is initiating minor compaction (all files) 2024-11-17T01:51:42,898 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/A is initiating minor compaction (all files) 2024-11-17T01:51:42,898 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/B in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:42,898 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/A in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:42,898 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/b308fdbfb044454e836770dad70478a2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/53a3dbf3769d425a93eeac7019195f5f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/d530213fbf794de494bdf1a12dc60e84] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=35.3 K 2024-11-17T01:51:42,898 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/2db4017dffcf40cfaebe5d24b4f9ac73, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/aba3b24898ca40e7bc8bbfbfbe1e956e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/9496a12c522c44b9941e1a47ac0b56fd] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=35.3 K 2024-11-17T01:51:42,899 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting b308fdbfb044454e836770dad70478a2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731808299989 2024-11-17T01:51:42,899 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2db4017dffcf40cfaebe5d24b4f9ac73, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731808299989 2024-11-17T01:51:42,901 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting aba3b24898ca40e7bc8bbfbfbe1e956e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731808301248 2024-11-17T01:51:42,901 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 53a3dbf3769d425a93eeac7019195f5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731808301248 2024-11-17T01:51:42,902 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting d530213fbf794de494bdf1a12dc60e84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731808301965 2024-11-17T01:51:42,902 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9496a12c522c44b9941e1a47ac0b56fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731808301965 2024-11-17T01:51:42,935 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:42,937 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/d8bcdd47aad743a3a07fcb7be36d59ed is 50, key is test_row_0/A:col10/1731808301965/Put/seqid=0 2024-11-17T01:51:42,945 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#B#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:42,946 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/084be17988404df0b2aa721b4395ca4d is 50, key is test_row_0/B:col10/1731808301965/Put/seqid=0 2024-11-17T01:51:42,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741855_1031 (size=12207) 2024-11-17T01:51:43,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741856_1032 (size=12207) 2024-11-17T01:51:43,007 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/d8bcdd47aad743a3a07fcb7be36d59ed as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/d8bcdd47aad743a3a07fcb7be36d59ed 2024-11-17T01:51:43,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T01:51:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:43,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:43,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:43,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:43,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:43,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:43,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:43,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/81cf77005e464700b2300a7b5dd6df18 is 50, key is test_row_0/A:col10/1731808302655/Put/seqid=0 2024-11-17T01:51:43,036 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/084be17988404df0b2aa721b4395ca4d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/084be17988404df0b2aa721b4395ca4d 2024-11-17T01:51:43,037 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/A of 5d8a196529c6f40b7321f17bb79ff0a8 into d8bcdd47aad743a3a07fcb7be36d59ed(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:43,037 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:43,037 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/A, priority=13, startTime=1731808302895; duration=0sec 2024-11-17T01:51:43,038 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:51:43,038 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:A 2024-11-17T01:51:43,038 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:51:43,043 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:51:43,044 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/C is initiating minor compaction (all files) 2024-11-17T01:51:43,044 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/C in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:43,044 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/02d397dd1904478fac6b69acbc3c40cc, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/85e7010e894d409dbea9ca4df13b4148, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/e5aff052a5f742c284d8fdac77b6ef07] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=35.3 K 2024-11-17T01:51:43,047 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02d397dd1904478fac6b69acbc3c40cc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731808299989 2024-11-17T01:51:43,048 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85e7010e894d409dbea9ca4df13b4148, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731808301248 2024-11-17T01:51:43,049 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5aff052a5f742c284d8fdac77b6ef07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731808301965 2024-11-17T01:51:43,060 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/B of 5d8a196529c6f40b7321f17bb79ff0a8 into 084be17988404df0b2aa721b4395ca4d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:43,060 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:43,060 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/B, priority=13, startTime=1731808302895; duration=0sec 2024-11-17T01:51:43,060 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:43,060 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:B 2024-11-17T01:51:43,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741857_1033 (size=16681) 2024-11-17T01:51:43,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/81cf77005e464700b2300a7b5dd6df18 2024-11-17T01:51:43,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808363055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,078 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#C#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:43,080 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/1cd4ed6dba8d4e459172f2f8006c1c04 is 50, key is test_row_0/C:col10/1731808301965/Put/seqid=0 2024-11-17T01:51:43,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808363057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808363061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808363073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808363065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/9d64877f27ca4949bd31786ea3245dc4 is 50, key is test_row_0/B:col10/1731808302655/Put/seqid=0 2024-11-17T01:51:43,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741858_1034 (size=12207) 2024-11-17T01:51:43,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741859_1035 (size=12001) 2024-11-17T01:51:43,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/9d64877f27ca4949bd31786ea3245dc4 2024-11-17T01:51:43,128 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/1cd4ed6dba8d4e459172f2f8006c1c04 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/1cd4ed6dba8d4e459172f2f8006c1c04 2024-11-17T01:51:43,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/ad7149306fee49ff9e3f660ab24400be is 50, key is test_row_0/C:col10/1731808302655/Put/seqid=0 2024-11-17T01:51:43,160 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/C of 5d8a196529c6f40b7321f17bb79ff0a8 into 1cd4ed6dba8d4e459172f2f8006c1c04(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:43,162 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:43,162 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/C, priority=13, startTime=1731808302895; duration=0sec 2024-11-17T01:51:43,162 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:43,163 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:C 2024-11-17T01:51:43,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808363174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741860_1036 (size=12001) 2024-11-17T01:51:43,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/ad7149306fee49ff9e3f660ab24400be 2024-11-17T01:51:43,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808363195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808363195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808363195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808363195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/81cf77005e464700b2300a7b5dd6df18 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/81cf77005e464700b2300a7b5dd6df18 2024-11-17T01:51:43,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/81cf77005e464700b2300a7b5dd6df18, entries=250, sequenceid=118, filesize=16.3 K 2024-11-17T01:51:43,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/9d64877f27ca4949bd31786ea3245dc4 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/9d64877f27ca4949bd31786ea3245dc4 2024-11-17T01:51:43,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/9d64877f27ca4949bd31786ea3245dc4, entries=150, sequenceid=118, filesize=11.7 K 2024-11-17T01:51:43,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/ad7149306fee49ff9e3f660ab24400be as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/ad7149306fee49ff9e3f660ab24400be 2024-11-17T01:51:43,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/ad7149306fee49ff9e3f660ab24400be, entries=150, sequenceid=118, filesize=11.7 K 2024-11-17T01:51:43,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 5d8a196529c6f40b7321f17bb79ff0a8 in 258ms, sequenceid=118, compaction requested=false 2024-11-17T01:51:43,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T01:51:43,396 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:51:43,396 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:51:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-17T01:51:43,400 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:51:43,402 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:51:43,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:51:43,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T01:51:43,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:43,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T01:51:43,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:43,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:43,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:43,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:43,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:43,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:43,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/c8640e1b126c41799fd65afe73f76154 is 50, key is test_row_0/A:col10/1731808303408/Put/seqid=0 2024-11-17T01:51:43,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741861_1037 (size=16881) 2024-11-17T01:51:43,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/c8640e1b126c41799fd65afe73f76154 2024-11-17T01:51:43,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/3f78dbb7652f47d2a275c6266594eed0 is 50, key is test_row_0/B:col10/1731808303408/Put/seqid=0 2024-11-17T01:51:43,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741862_1038 (size=12151) 2024-11-17T01:51:43,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/3f78dbb7652f47d2a275c6266594eed0 2024-11-17T01:51:43,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/847a767fd3e444e0855eeaa8e6e51ace is 50, key is test_row_0/C:col10/1731808303408/Put/seqid=0 2024-11-17T01:51:43,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T01:51:43,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741863_1039 (size=12151) 2024-11-17T01:51:43,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/847a767fd3e444e0855eeaa8e6e51ace 2024-11-17T01:51:43,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/c8640e1b126c41799fd65afe73f76154 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/c8640e1b126c41799fd65afe73f76154 2024-11-17T01:51:43,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/c8640e1b126c41799fd65afe73f76154, entries=250, sequenceid=137, filesize=16.5 K 2024-11-17T01:51:43,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/3f78dbb7652f47d2a275c6266594eed0 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/3f78dbb7652f47d2a275c6266594eed0 2024-11-17T01:51:43,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T01:51:43,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:43,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:43,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:43,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:43,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808363535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:43,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808363544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/3f78dbb7652f47d2a275c6266594eed0, entries=150, sequenceid=137, filesize=11.9 K 2024-11-17T01:51:43,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808363549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/847a767fd3e444e0855eeaa8e6e51ace as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/847a767fd3e444e0855eeaa8e6e51ace 2024-11-17T01:51:43,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808363560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/847a767fd3e444e0855eeaa8e6e51ace, entries=150, sequenceid=137, filesize=11.9 K 2024-11-17T01:51:43,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808363557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5d8a196529c6f40b7321f17bb79ff0a8 in 167ms, sequenceid=137, compaction requested=true 2024-11-17T01:51:43,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:43,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,584 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:51:43,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:51:43,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:43,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:51:43,586 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:51:43,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:43,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:51:43,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:51:43,587 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45769 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:51:43,587 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/A is initiating minor compaction (all files) 2024-11-17T01:51:43,588 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/A in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:43,588 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/d8bcdd47aad743a3a07fcb7be36d59ed, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/81cf77005e464700b2300a7b5dd6df18, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/c8640e1b126c41799fd65afe73f76154] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=44.7 K 2024-11-17T01:51:43,589 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:51:43,589 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8bcdd47aad743a3a07fcb7be36d59ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731808301965 2024-11-17T01:51:43,589 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/B is initiating minor compaction (all files) 2024-11-17T01:51:43,589 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/B in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:43,589 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/084be17988404df0b2aa721b4395ca4d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/9d64877f27ca4949bd31786ea3245dc4, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/3f78dbb7652f47d2a275c6266594eed0] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=35.5 K 2024-11-17T01:51:43,589 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81cf77005e464700b2300a7b5dd6df18, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731808302655 2024-11-17T01:51:43,590 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 084be17988404df0b2aa721b4395ca4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731808301965 2024-11-17T01:51:43,590 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8640e1b126c41799fd65afe73f76154, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731808303048 2024-11-17T01:51:43,592 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d64877f27ca4949bd31786ea3245dc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731808302655 2024-11-17T01:51:43,593 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f78dbb7652f47d2a275c6266594eed0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731808303060 2024-11-17T01:51:43,622 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#A#compaction#27 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:43,623 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/a0d60219b8c7466981d465acae7a8015 is 50, key is test_row_0/A:col10/1731808303408/Put/seqid=0 2024-11-17T01:51:43,638 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#B#compaction#28 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:43,639 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/5aff1b7e6d9a47e9858b77a40e885ef8 is 50, key is test_row_0/B:col10/1731808303408/Put/seqid=0 2024-11-17T01:51:43,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741864_1040 (size=12459) 2024-11-17T01:51:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:43,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T01:51:43,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:43,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:43,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:43,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:43,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:43,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:43,689 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/a0d60219b8c7466981d465acae7a8015 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/a0d60219b8c7466981d465acae7a8015 2024-11-17T01:51:43,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741865_1041 (size=12459) 2024-11-17T01:51:43,703 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/A of 5d8a196529c6f40b7321f17bb79ff0a8 into a0d60219b8c7466981d465acae7a8015(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:43,703 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:43,703 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/A, priority=13, startTime=1731808303584; duration=0sec 2024-11-17T01:51:43,703 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:51:43,704 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:A 2024-11-17T01:51:43,704 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:51:43,706 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:51:43,711 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/C is initiating minor compaction (all files) 2024-11-17T01:51:43,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/8a8fa05a3bd94235afb63b9e64c8ea23 is 50, key is test_row_0/A:col10/1731808303683/Put/seqid=0 2024-11-17T01:51:43,714 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/C in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:43,715 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/1cd4ed6dba8d4e459172f2f8006c1c04, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/ad7149306fee49ff9e3f660ab24400be, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/847a767fd3e444e0855eeaa8e6e51ace] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=35.5 K 2024-11-17T01:51:43,717 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cd4ed6dba8d4e459172f2f8006c1c04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731808301965 2024-11-17T01:51:43,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T01:51:43,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:43,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:43,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:43,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:43,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:43,720 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad7149306fee49ff9e3f660ab24400be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731808302655 2024-11-17T01:51:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:43,721 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 847a767fd3e444e0855eeaa8e6e51ace, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731808303060 2024-11-17T01:51:43,721 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/5aff1b7e6d9a47e9858b77a40e885ef8 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/5aff1b7e6d9a47e9858b77a40e885ef8 2024-11-17T01:51:43,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741866_1042 (size=14541) 2024-11-17T01:51:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T01:51:43,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/8a8fa05a3bd94235afb63b9e64c8ea23 2024-11-17T01:51:43,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808363738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808363744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808363746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808363774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,786 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/B of 5d8a196529c6f40b7321f17bb79ff0a8 into 5aff1b7e6d9a47e9858b77a40e885ef8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:43,786 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:43,786 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/B, priority=13, startTime=1731808303585; duration=0sec 2024-11-17T01:51:43,786 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:43,786 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:B 2024-11-17T01:51:43,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808363776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,795 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#C#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:43,795 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/29004c73b687472bb2e826fddd121fd1 is 50, key is test_row_0/C:col10/1731808303408/Put/seqid=0 2024-11-17T01:51:43,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/8456ecc9dca34c4e840bebf18e8ace7e is 50, key is test_row_0/B:col10/1731808303683/Put/seqid=0 2024-11-17T01:51:43,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741868_1044 (size=12151) 2024-11-17T01:51:43,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/8456ecc9dca34c4e840bebf18e8ace7e 2024-11-17T01:51:43,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741867_1043 (size=12459) 2024-11-17T01:51:43,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T01:51:43,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:43,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:43,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:43,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:43,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:43,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808363875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,888 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/29004c73b687472bb2e826fddd121fd1 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/29004c73b687472bb2e826fddd121fd1 2024-11-17T01:51:43,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/84c6c532d75e4cc99f7646817a6fdbef is 50, key is test_row_0/C:col10/1731808303683/Put/seqid=0 2024-11-17T01:51:43,902 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/C of 5d8a196529c6f40b7321f17bb79ff0a8 into 29004c73b687472bb2e826fddd121fd1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:43,902 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:43,902 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/C, priority=13, startTime=1731808303586; duration=0sec 2024-11-17T01:51:43,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,902 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808363885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,902 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:C 2024-11-17T01:51:43,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808363885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808363886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:43,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808363894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:43,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:43,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:43,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741869_1045 (size=12151) 2024-11-17T01:51:43,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/84c6c532d75e4cc99f7646817a6fdbef 2024-11-17T01:51:43,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/8a8fa05a3bd94235afb63b9e64c8ea23 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/8a8fa05a3bd94235afb63b9e64c8ea23 2024-11-17T01:51:43,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/8a8fa05a3bd94235afb63b9e64c8ea23, entries=200, sequenceid=154, filesize=14.2 K 2024-11-17T01:51:43,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/8456ecc9dca34c4e840bebf18e8ace7e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/8456ecc9dca34c4e840bebf18e8ace7e 2024-11-17T01:51:43,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/8456ecc9dca34c4e840bebf18e8ace7e, entries=150, sequenceid=154, filesize=11.9 K 2024-11-17T01:51:43,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/84c6c532d75e4cc99f7646817a6fdbef as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/84c6c532d75e4cc99f7646817a6fdbef 2024-11-17T01:51:44,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/84c6c532d75e4cc99f7646817a6fdbef, entries=150, sequenceid=154, filesize=11.9 K 2024-11-17T01:51:44,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 5d8a196529c6f40b7321f17bb79ff0a8 in 332ms, sequenceid=154, compaction requested=false 2024-11-17T01:51:44,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T01:51:44,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T01:51:44,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:44,040 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T01:51:44,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:44,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:44,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:44,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:44,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:44,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:44,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/b9df8a84627849f6b468e695fad70f00 is 50, key is test_row_0/A:col10/1731808303741/Put/seqid=0 2024-11-17T01:51:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:44,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:44,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741870_1046 (size=12151) 2024-11-17T01:51:44,110 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/b9df8a84627849f6b468e695fad70f00 2024-11-17T01:51:44,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/2ee60e53af9041de8f6aeb19033ec359 is 50, key is test_row_0/B:col10/1731808303741/Put/seqid=0 2024-11-17T01:51:44,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741871_1047 (size=12151) 2024-11-17T01:51:44,149 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/2ee60e53af9041de8f6aeb19033ec359 2024-11-17T01:51:44,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808364141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:44,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808364142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808364149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:44,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808364141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:44,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808364160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:44,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/9a9055b691a44c36b88c931c276e69fb is 50, key is test_row_0/C:col10/1731808303741/Put/seqid=0 2024-11-17T01:51:44,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741872_1048 (size=12151) 2024-11-17T01:51:44,223 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/9a9055b691a44c36b88c931c276e69fb 2024-11-17T01:51:44,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/b9df8a84627849f6b468e695fad70f00 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/b9df8a84627849f6b468e695fad70f00 2024-11-17T01:51:44,255 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/b9df8a84627849f6b468e695fad70f00, entries=150, sequenceid=173, filesize=11.9 K 2024-11-17T01:51:44,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/2ee60e53af9041de8f6aeb19033ec359 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/2ee60e53af9041de8f6aeb19033ec359 2024-11-17T01:51:44,271 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/2ee60e53af9041de8f6aeb19033ec359, entries=150, sequenceid=173, filesize=11.9 K 2024-11-17T01:51:44,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808364266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:44,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808364266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:44,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/9a9055b691a44c36b88c931c276e69fb as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/9a9055b691a44c36b88c931c276e69fb 2024-11-17T01:51:44,283 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/9a9055b691a44c36b88c931c276e69fb, entries=150, sequenceid=173, filesize=11.9 K 2024-11-17T01:51:44,285 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5d8a196529c6f40b7321f17bb79ff0a8 in 245ms, sequenceid=173, compaction requested=true 2024-11-17T01:51:44,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:44,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:44,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-17T01:51:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-17T01:51:44,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-17T01:51:44,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 888 msec 2024-11-17T01:51:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:44,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T01:51:44,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:44,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:44,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:44,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:44,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:44,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:44,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/0461162c235148abbe730e2eafecba60 is 50, key is test_row_0/A:col10/1731808304156/Put/seqid=0 2024-11-17T01:51:44,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 900 msec 2024-11-17T01:51:44,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741873_1049 (size=14541) 2024-11-17T01:51:44,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808364354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:44,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808364362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:44,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:44,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808364362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:44,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:44,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808364475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:44,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808364475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:44,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:44,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808364485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:44,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808364485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:44,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808364485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:44,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:44,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:44,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T01:51:44,545 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:51:44,545 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:51:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-17T01:51:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T01:51:44,552 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:51:44,553 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:51:44,553 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:51:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T01:51:44,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808364694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:44,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808364694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:44,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808364704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:44,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T01:51:44,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:44,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:44,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:44,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:44,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:44,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/0461162c235148abbe730e2eafecba60 2024-11-17T01:51:44,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/b6971f7259fb49daa379b0e79dedffa3 is 50, key is test_row_0/B:col10/1731808304156/Put/seqid=0 2024-11-17T01:51:44,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741874_1050 (size=12151) 2024-11-17T01:51:44,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/b6971f7259fb49daa379b0e79dedffa3 2024-11-17T01:51:44,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/2b79946d6b974a08aeceb7ff6f167508 is 50, key is test_row_0/C:col10/1731808304156/Put/seqid=0 2024-11-17T01:51:44,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:44,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808364805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:44,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:44,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808364805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:44,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741875_1051 (size=12151) 2024-11-17T01:51:44,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T01:51:44,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:44,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:44,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:44,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:44,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:44,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:44,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T01:51:45,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:45,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808365005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:45,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:45,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808365005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:45,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:45,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:45,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T01:51:45,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:45,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:45,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:45,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:45,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:45,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:45,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808365014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:45,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:45,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T01:51:45,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:45,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:45,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:45,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:45,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T01:51:45,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/2b79946d6b974a08aeceb7ff6f167508 2024-11-17T01:51:45,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/0461162c235148abbe730e2eafecba60 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/0461162c235148abbe730e2eafecba60 2024-11-17T01:51:45,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/0461162c235148abbe730e2eafecba60, entries=200, sequenceid=190, filesize=14.2 K 2024-11-17T01:51:45,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/b6971f7259fb49daa379b0e79dedffa3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/b6971f7259fb49daa379b0e79dedffa3 2024-11-17T01:51:45,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/b6971f7259fb49daa379b0e79dedffa3, entries=150, sequenceid=190, filesize=11.9 K 2024-11-17T01:51:45,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/2b79946d6b974a08aeceb7ff6f167508 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/2b79946d6b974a08aeceb7ff6f167508 2024-11-17T01:51:45,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/2b79946d6b974a08aeceb7ff6f167508, entries=150, sequenceid=190, filesize=11.9 K 2024-11-17T01:51:45,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 5d8a196529c6f40b7321f17bb79ff0a8 in 969ms, sequenceid=190, compaction requested=true 2024-11-17T01:51:45,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:45,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:51:45,264 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:51:45,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:45,264 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:51:45,267 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53692 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:51:45,267 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:51:45,267 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/B is initiating minor compaction (all files) 2024-11-17T01:51:45,267 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/A is initiating minor compaction (all files) 2024-11-17T01:51:45,267 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/A in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:45,267 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/a0d60219b8c7466981d465acae7a8015, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/8a8fa05a3bd94235afb63b9e64c8ea23, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/b9df8a84627849f6b468e695fad70f00, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/0461162c235148abbe730e2eafecba60] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=52.4 K 2024-11-17T01:51:45,268 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/B in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:45,268 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/5aff1b7e6d9a47e9858b77a40e885ef8, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/8456ecc9dca34c4e840bebf18e8ace7e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/2ee60e53af9041de8f6aeb19033ec359, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/b6971f7259fb49daa379b0e79dedffa3] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=47.8 K 2024-11-17T01:51:45,269 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0d60219b8c7466981d465acae7a8015, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731808303060 2024-11-17T01:51:45,269 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aff1b7e6d9a47e9858b77a40e885ef8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731808303060 2024-11-17T01:51:45,270 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a8fa05a3bd94235afb63b9e64c8ea23, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731808303525 2024-11-17T01:51:45,270 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 8456ecc9dca34c4e840bebf18e8ace7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731808303544 2024-11-17T01:51:45,271 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9df8a84627849f6b468e695fad70f00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731808303700 2024-11-17T01:51:45,272 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ee60e53af9041de8f6aeb19033ec359, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731808303700 2024-11-17T01:51:45,272 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting b6971f7259fb49daa379b0e79dedffa3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731808304138 2024-11-17T01:51:45,273 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0461162c235148abbe730e2eafecba60, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731808304138 2024-11-17T01:51:45,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:51:45,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:45,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:51:45,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:51:45,298 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#A#compaction#39 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:45,299 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/1052f0f05bc24a7d8a8e6e33cd884880 is 50, key is test_row_0/A:col10/1731808304156/Put/seqid=0 2024-11-17T01:51:45,302 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:45,303 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/d6e2058286fb47d2968faac70f2821e7 is 50, key is test_row_0/B:col10/1731808304156/Put/seqid=0 2024-11-17T01:51:45,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741876_1052 (size=12595) 2024-11-17T01:51:45,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:45,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T01:51:45,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:45,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:45,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T01:51:45,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:45,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:45,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:45,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:45,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:45,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:45,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:45,332 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/1052f0f05bc24a7d8a8e6e33cd884880 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/1052f0f05bc24a7d8a8e6e33cd884880 2024-11-17T01:51:45,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:45,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:45,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741877_1053 (size=12595) 2024-11-17T01:51:45,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/ed0ea1ae2aa24348bd194981e7a181cd is 50, key is test_row_0/A:col10/1731808305326/Put/seqid=0 2024-11-17T01:51:45,353 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/A of 5d8a196529c6f40b7321f17bb79ff0a8 into 1052f0f05bc24a7d8a8e6e33cd884880(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:45,354 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:45,354 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/A, priority=12, startTime=1731808305264; duration=0sec 2024-11-17T01:51:45,354 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:51:45,354 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:A 2024-11-17T01:51:45,354 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:51:45,354 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/d6e2058286fb47d2968faac70f2821e7 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/d6e2058286fb47d2968faac70f2821e7 2024-11-17T01:51:45,356 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:51:45,356 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/C is initiating minor compaction (all files) 2024-11-17T01:51:45,356 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/C in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:45,357 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/29004c73b687472bb2e826fddd121fd1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/84c6c532d75e4cc99f7646817a6fdbef, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/9a9055b691a44c36b88c931c276e69fb, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/2b79946d6b974a08aeceb7ff6f167508] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=47.8 K 2024-11-17T01:51:45,357 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29004c73b687472bb2e826fddd121fd1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731808303060 2024-11-17T01:51:45,358 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84c6c532d75e4cc99f7646817a6fdbef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731808303544 2024-11-17T01:51:45,358 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a9055b691a44c36b88c931c276e69fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731808303700 2024-11-17T01:51:45,359 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b79946d6b974a08aeceb7ff6f167508, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731808304138 2024-11-17T01:51:45,368 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/B of 5d8a196529c6f40b7321f17bb79ff0a8 into d6e2058286fb47d2968faac70f2821e7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:45,368 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:45,368 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/B, priority=12, startTime=1731808305264; duration=0sec 2024-11-17T01:51:45,369 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:45,369 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:B 2024-11-17T01:51:45,378 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:45,379 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/9ccafd59389148bb92a9fb1b05796d15 is 50, key is test_row_0/C:col10/1731808304156/Put/seqid=0 2024-11-17T01:51:45,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741878_1054 (size=14541) 2024-11-17T01:51:45,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/ed0ea1ae2aa24348bd194981e7a181cd 2024-11-17T01:51:45,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/ab99064ee7734320897342da85446e0a is 50, key is test_row_0/B:col10/1731808305326/Put/seqid=0 2024-11-17T01:51:45,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:45,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808365405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:45,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:45,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:45,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808365407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:45,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:45,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741879_1055 (size=12595) 2024-11-17T01:51:45,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741880_1056 (size=12151) 2024-11-17T01:51:45,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/ab99064ee7734320897342da85446e0a 2024-11-17T01:51:45,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/7147c661c7e34cc1952fabc49120d9c1 is 50, key is test_row_0/C:col10/1731808305326/Put/seqid=0 2024-11-17T01:51:45,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T01:51:45,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:45,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:45,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:45,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:45,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:45,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741881_1057 (size=12151) 2024-11-17T01:51:45,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/7147c661c7e34cc1952fabc49120d9c1 2024-11-17T01:51:45,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/ed0ea1ae2aa24348bd194981e7a181cd as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/ed0ea1ae2aa24348bd194981e7a181cd 2024-11-17T01:51:45,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/ed0ea1ae2aa24348bd194981e7a181cd, entries=200, sequenceid=209, filesize=14.2 K 2024-11-17T01:51:45,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/ab99064ee7734320897342da85446e0a as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/ab99064ee7734320897342da85446e0a 2024-11-17T01:51:45,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808365514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:45,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808365524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:45,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:45,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808365525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808365525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:45,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/ab99064ee7734320897342da85446e0a, entries=150, sequenceid=209, filesize=11.9 K 2024-11-17T01:51:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:45,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:45,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/7147c661c7e34cc1952fabc49120d9c1 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/7147c661c7e34cc1952fabc49120d9c1 2024-11-17T01:51:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808365534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:45,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:45,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/7147c661c7e34cc1952fabc49120d9c1, entries=150, sequenceid=209, filesize=11.9 K 2024-11-17T01:51:45,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5d8a196529c6f40b7321f17bb79ff0a8 in 216ms, sequenceid=209, compaction requested=false 2024-11-17T01:51:45,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:45,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T01:51:45,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:45,650 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T01:51:45,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:45,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:45,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:45,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:45,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:45,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:45,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/4c2d9d6124d941128d5dfc801e6d2eed is 50, key is test_row_0/A:col10/1731808305401/Put/seqid=0 2024-11-17T01:51:45,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741882_1058 (size=12151) 2024-11-17T01:51:45,677 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/4c2d9d6124d941128d5dfc801e6d2eed 2024-11-17T01:51:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T01:51:45,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/3c33cd4b1c764c29962076c230a8ba99 is 50, key is test_row_0/B:col10/1731808305401/Put/seqid=0 2024-11-17T01:51:45,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741883_1059 (size=12151) 2024-11-17T01:51:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:45,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:45,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808365803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:45,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:45,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808365810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:45,855 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/9ccafd59389148bb92a9fb1b05796d15 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/9ccafd59389148bb92a9fb1b05796d15 2024-11-17T01:51:45,873 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/C of 5d8a196529c6f40b7321f17bb79ff0a8 into 9ccafd59389148bb92a9fb1b05796d15(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:45,873 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:45,874 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/C, priority=12, startTime=1731808305293; duration=0sec 2024-11-17T01:51:45,874 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:45,874 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:C 2024-11-17T01:51:45,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808365925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:45,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808365925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:45,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:45,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:46,117 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/3c33cd4b1c764c29962076c230a8ba99 2024-11-17T01:51:46,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/b80738aeef2044ef8a74b5df84c89061 is 50, key is test_row_0/C:col10/1731808305401/Put/seqid=0 2024-11-17T01:51:46,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808366135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:46,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:46,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808366135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:46,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:46,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:46,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741884_1060 (size=12151) 2024-11-17T01:51:46,159 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/b80738aeef2044ef8a74b5df84c89061 2024-11-17T01:51:46,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/4c2d9d6124d941128d5dfc801e6d2eed as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/4c2d9d6124d941128d5dfc801e6d2eed 2024-11-17T01:51:46,178 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/4c2d9d6124d941128d5dfc801e6d2eed, entries=150, sequenceid=224, filesize=11.9 K 2024-11-17T01:51:46,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/3c33cd4b1c764c29962076c230a8ba99 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/3c33cd4b1c764c29962076c230a8ba99 2024-11-17T01:51:46,192 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/3c33cd4b1c764c29962076c230a8ba99, entries=150, sequenceid=224, filesize=11.9 K 2024-11-17T01:51:46,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/b80738aeef2044ef8a74b5df84c89061 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/b80738aeef2044ef8a74b5df84c89061 2024-11-17T01:51:46,218 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/b80738aeef2044ef8a74b5df84c89061, entries=150, sequenceid=224, filesize=11.9 K 2024-11-17T01:51:46,221 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5d8a196529c6f40b7321f17bb79ff0a8 in 571ms, sequenceid=224, compaction requested=true 2024-11-17T01:51:46,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:46,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:46,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-17T01:51:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-17T01:51:46,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-17T01:51:46,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6710 sec 2024-11-17T01:51:46,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.6870 sec 2024-11-17T01:51:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:46,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-11-17T01:51:46,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808305789, max timestamp=1731808306453Num uniques 859; 2024-11-17T01:51:46,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:51:46,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808305789, max timestamp=1731808306453Num uniques 859; 2024-11-17T01:51:46,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:51:46,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:46,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:51:46,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:46,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:51:46,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:46,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:51:46,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/ae583273dd82464087f178ce35e4aa42 is 50, key is test_row_0/A:col10/1731808306451/Put/seqid=0 2024-11-17T01:51:46,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741885_1061 (size=14541) 2024-11-17T01:51:46,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808306539, max timestamp=1731808306553Num uniques 859; 2024-11-17T01:51:46,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:51:46,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808306539, max timestamp=1731808306553Num uniques 859; 2024-11-17T01:51:46,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:51:46,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808306539, max timestamp=1731808306553Num uniques 859; 2024-11-17T01:51:46,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:51:46,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808366633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:46,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:46,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808366634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:46,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808366636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:46,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:46,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:46,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808366636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:46,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:46,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808366647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:46,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T01:51:46,694 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:51:46,695 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:51:46,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-17T01:51:46,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T01:51:46,699 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:51:46,700 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:51:46,700 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:51:46,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808366754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808366755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808366755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808366756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:46,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808366764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:46,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T01:51:46,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T01:51:46,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:46,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:46,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:46,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:46,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:46,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:46,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/ae583273dd82464087f178ce35e4aa42 2024-11-17T01:51:46,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/45637b972ad1406e90c9aee1842f5937 is 50, key is test_row_0/B:col10/1731808306451/Put/seqid=0 2024-11-17T01:51:46,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:46,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:46,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808366964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:46,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808366964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:46,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808366965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:46,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:46,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808366965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:46,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:46,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808366974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:46,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741886_1062 (size=12151) 2024-11-17T01:51:46,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/45637b972ad1406e90c9aee1842f5937 2024-11-17T01:51:47,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T01:51:47,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:47,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:47,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:47,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:47,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:47,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/924514e3957f4899bae2e3f14431ee6e is 50, key is test_row_0/C:col10/1731808306451/Put/seqid=0 2024-11-17T01:51:47,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:47,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T01:51:47,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741887_1063 (size=12151) 2024-11-17T01:51:47,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/924514e3957f4899bae2e3f14431ee6e 2024-11-17T01:51:47,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/ae583273dd82464087f178ce35e4aa42 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/ae583273dd82464087f178ce35e4aa42 2024-11-17T01:51:47,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/ae583273dd82464087f178ce35e4aa42, entries=200, sequenceid=246, filesize=14.2 K 2024-11-17T01:51:47,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/45637b972ad1406e90c9aee1842f5937 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/45637b972ad1406e90c9aee1842f5937 2024-11-17T01:51:47,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/45637b972ad1406e90c9aee1842f5937, entries=150, sequenceid=246, filesize=11.9 K 2024-11-17T01:51:47,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/924514e3957f4899bae2e3f14431ee6e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/924514e3957f4899bae2e3f14431ee6e 2024-11-17T01:51:47,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/924514e3957f4899bae2e3f14431ee6e, entries=150, sequenceid=246, filesize=11.9 K 2024-11-17T01:51:47,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for 5d8a196529c6f40b7321f17bb79ff0a8 in 635ms, sequenceid=246, compaction requested=true 2024-11-17T01:51:47,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:47,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:51:47,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:47,090 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:51:47,091 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:51:47,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:51:47,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:47,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:51:47,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:51:47,092 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:51:47,093 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/A is initiating minor compaction (all files) 2024-11-17T01:51:47,093 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:51:47,093 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/A in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:47,093 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/B is initiating minor compaction (all files) 2024-11-17T01:51:47,093 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/B in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:47,093 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/1052f0f05bc24a7d8a8e6e33cd884880, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/ed0ea1ae2aa24348bd194981e7a181cd, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/4c2d9d6124d941128d5dfc801e6d2eed, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/ae583273dd82464087f178ce35e4aa42] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=52.6 K 2024-11-17T01:51:47,093 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/d6e2058286fb47d2968faac70f2821e7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/ab99064ee7734320897342da85446e0a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/3c33cd4b1c764c29962076c230a8ba99, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/45637b972ad1406e90c9aee1842f5937] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=47.9 K 2024-11-17T01:51:47,094 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1052f0f05bc24a7d8a8e6e33cd884880, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731808304138 2024-11-17T01:51:47,094 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting d6e2058286fb47d2968faac70f2821e7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731808304138 2024-11-17T01:51:47,094 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed0ea1ae2aa24348bd194981e7a181cd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731808304359 2024-11-17T01:51:47,095 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting ab99064ee7734320897342da85446e0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731808305320 2024-11-17T01:51:47,095 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c2d9d6124d941128d5dfc801e6d2eed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731808305399 2024-11-17T01:51:47,095 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c33cd4b1c764c29962076c230a8ba99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731808305399 2024-11-17T01:51:47,095 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae583273dd82464087f178ce35e4aa42, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731808305789 2024-11-17T01:51:47,095 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 45637b972ad1406e90c9aee1842f5937, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731808305808 2024-11-17T01:51:47,126 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#B#compaction#51 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:47,127 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/7528cef67934485a9743052ccf4ff7aa is 50, key is test_row_0/B:col10/1731808306451/Put/seqid=0 2024-11-17T01:51:47,133 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#A#compaction#52 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:47,134 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/f0163584b9f64614b1fa2ab4ce39fb2d is 50, key is test_row_0/A:col10/1731808306451/Put/seqid=0 2024-11-17T01:51:47,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741888_1064 (size=12731) 2024-11-17T01:51:47,159 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/7528cef67934485a9743052ccf4ff7aa as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/7528cef67934485a9743052ccf4ff7aa 2024-11-17T01:51:47,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T01:51:47,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:47,164 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-17T01:51:47,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:47,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:51:47,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:47,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:51:47,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:47,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:51:47,173 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/B of 5d8a196529c6f40b7321f17bb79ff0a8 into 7528cef67934485a9743052ccf4ff7aa(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:47,173 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:47,173 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/B, priority=12, startTime=1731808307090; duration=0sec 2024-11-17T01:51:47,174 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:51:47,174 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:B 2024-11-17T01:51:47,174 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:51:47,177 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:51:47,177 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/C is initiating minor compaction (all files) 2024-11-17T01:51:47,177 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/C in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:47,177 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/9ccafd59389148bb92a9fb1b05796d15, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/7147c661c7e34cc1952fabc49120d9c1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/b80738aeef2044ef8a74b5df84c89061, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/924514e3957f4899bae2e3f14431ee6e] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=47.9 K 2024-11-17T01:51:47,178 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ccafd59389148bb92a9fb1b05796d15, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731808304138 2024-11-17T01:51:47,181 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 7147c661c7e34cc1952fabc49120d9c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731808305320 2024-11-17T01:51:47,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/d8a1d3f7549447c2bf52d445730e6712 is 50, key is test_row_0/A:col10/1731808306629/Put/seqid=0 2024-11-17T01:51:47,183 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting b80738aeef2044ef8a74b5df84c89061, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731808305399 2024-11-17T01:51:47,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741889_1065 (size=12731) 2024-11-17T01:51:47,184 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 924514e3957f4899bae2e3f14431ee6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731808305808 2024-11-17T01:51:47,190 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/f0163584b9f64614b1fa2ab4ce39fb2d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/f0163584b9f64614b1fa2ab4ce39fb2d 2024-11-17T01:51:47,202 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#C#compaction#54 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:47,203 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/fd8273e4810348d3bb84bd3bffaacc96 is 50, key is test_row_0/C:col10/1731808306451/Put/seqid=0 2024-11-17T01:51:47,206 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/A of 5d8a196529c6f40b7321f17bb79ff0a8 into f0163584b9f64614b1fa2ab4ce39fb2d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:47,206 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:47,206 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/A, priority=12, startTime=1731808307090; duration=0sec 2024-11-17T01:51:47,206 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:47,206 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:A 2024-11-17T01:51:47,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741890_1066 (size=12301) 2024-11-17T01:51:47,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741891_1067 (size=12731) 2024-11-17T01:51:47,248 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/fd8273e4810348d3bb84bd3bffaacc96 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/fd8273e4810348d3bb84bd3bffaacc96 2024-11-17T01:51:47,256 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/C of 5d8a196529c6f40b7321f17bb79ff0a8 into fd8273e4810348d3bb84bd3bffaacc96(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:47,256 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:47,256 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/C, priority=12, startTime=1731808307091; duration=0sec 2024-11-17T01:51:47,257 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:47,257 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:C 2024-11-17T01:51:47,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:47,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:47,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808367286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:47,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:47,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808367290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:47,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:47,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808367292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:47,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:47,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808367292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:47,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:47,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808367294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:47,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T01:51:47,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808367404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808367404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:47,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808367414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:47,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:47,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808367415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:47,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:47,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:47,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808367415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:47,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:47,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:47,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808367614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808367614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:47,618 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/d8a1d3f7549447c2bf52d445730e6712 2024-11-17T01:51:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:47,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808367624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:47,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808367625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808367626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:47,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:47,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/130a4066353c46b7a7351041ca724c4c is 50, key is test_row_0/B:col10/1731808306629/Put/seqid=0 2024-11-17T01:51:47,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741892_1068 (size=12301) 2024-11-17T01:51:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T01:51:47,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808367925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808367925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:47,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808367934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:47,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808367934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808367934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:47,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:47,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:47,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:48,040 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/130a4066353c46b7a7351041ca724c4c 2024-11-17T01:51:48,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/40f3c7302def4ec1b12687de81233114 is 50, key is test_row_0/C:col10/1731808306629/Put/seqid=0 2024-11-17T01:51:48,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741893_1069 (size=12301) 2024-11-17T01:51:48,077 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/40f3c7302def4ec1b12687de81233114 2024-11-17T01:51:48,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/d8a1d3f7549447c2bf52d445730e6712 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/d8a1d3f7549447c2bf52d445730e6712 2024-11-17T01:51:48,105 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/d8a1d3f7549447c2bf52d445730e6712, entries=150, sequenceid=282, filesize=12.0 K 2024-11-17T01:51:48,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/130a4066353c46b7a7351041ca724c4c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/130a4066353c46b7a7351041ca724c4c 2024-11-17T01:51:48,116 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/130a4066353c46b7a7351041ca724c4c, entries=150, sequenceid=282, filesize=12.0 K 2024-11-17T01:51:48,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/40f3c7302def4ec1b12687de81233114 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/40f3c7302def4ec1b12687de81233114 2024-11-17T01:51:48,146 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/40f3c7302def4ec1b12687de81233114, entries=150, sequenceid=282, filesize=12.0 K 2024-11-17T01:51:48,167 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 5d8a196529c6f40b7321f17bb79ff0a8 in 1003ms, sequenceid=282, compaction requested=false 2024-11-17T01:51:48,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:48,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:48,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-17T01:51:48,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-17T01:51:48,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-17T01:51:48,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4710 sec 2024-11-17T01:51:48,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.4830 sec 2024-11-17T01:51:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:48,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T01:51:48,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:48,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:48,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:48,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:48,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:48,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:48,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/cd8b09283d704b3791deccbbc2bc2612 is 50, key is test_row_0/A:col10/1731808307283/Put/seqid=0 2024-11-17T01:51:48,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741894_1070 (size=14741) 2024-11-17T01:51:48,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808368471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808368471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:48,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:48,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808368473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:48,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:48,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808368475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:48,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:48,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808368481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:48,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:48,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808368585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:48,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:48,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808368595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808368595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:48,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:48,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:48,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:48,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808368595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808368595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:48,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808368794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:48,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808368804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:48,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808368804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:48,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808368804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:48,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:48,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:48,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808368804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:48,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:48,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T01:51:48,844 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:51:48,845 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:51:48,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-17T01:51:48,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T01:51:48,847 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:51:48,848 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:51:48,848 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:51:48,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/cd8b09283d704b3791deccbbc2bc2612 2024-11-17T01:51:48,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/ae46911fdac34497ae101874d1b83677 is 50, key is test_row_0/B:col10/1731808307283/Put/seqid=0 2024-11-17T01:51:48,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741895_1071 (size=12301) 2024-11-17T01:51:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T01:51:49,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T01:51:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:49,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:49,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808369104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:49,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:49,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:49,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808369114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:49,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:49,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808369114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:49,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:49,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808369114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:49,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808369114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:49,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T01:51:49,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:49,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:49,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:49,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:49,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:49,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:49,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T01:51:49,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/ae46911fdac34497ae101874d1b83677 2024-11-17T01:51:49,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/409ad773707d4631b63772e14ea846a0 is 50, key is test_row_0/C:col10/1731808307283/Put/seqid=0 2024-11-17T01:51:49,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T01:51:49,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:49,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:49,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:49,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:49,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:49,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:49,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741896_1072 (size=12301) 2024-11-17T01:51:49,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/409ad773707d4631b63772e14ea846a0 2024-11-17T01:51:49,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/cd8b09283d704b3791deccbbc2bc2612 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/cd8b09283d704b3791deccbbc2bc2612 2024-11-17T01:51:49,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/cd8b09283d704b3791deccbbc2bc2612, entries=200, sequenceid=297, filesize=14.4 K 2024-11-17T01:51:49,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/ae46911fdac34497ae101874d1b83677 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/ae46911fdac34497ae101874d1b83677 2024-11-17T01:51:49,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/ae46911fdac34497ae101874d1b83677, entries=150, sequenceid=297, filesize=12.0 K 2024-11-17T01:51:49,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/409ad773707d4631b63772e14ea846a0 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/409ad773707d4631b63772e14ea846a0 2024-11-17T01:51:49,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/409ad773707d4631b63772e14ea846a0, entries=150, sequenceid=297, filesize=12.0 K 2024-11-17T01:51:49,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5d8a196529c6f40b7321f17bb79ff0a8 in 942ms, sequenceid=297, compaction requested=true 2024-11-17T01:51:49,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:49,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:51:49,378 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:51:49,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:49,378 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:51:49,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:51:49,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:49,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:51:49,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:51:49,379 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:51:49,379 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:51:49,380 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/A is initiating minor compaction (all files) 2024-11-17T01:51:49,380 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/B is initiating minor compaction (all files) 2024-11-17T01:51:49,382 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/A in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:49,382 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/B in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:49,382 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/f0163584b9f64614b1fa2ab4ce39fb2d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/d8a1d3f7549447c2bf52d445730e6712, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/cd8b09283d704b3791deccbbc2bc2612] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=38.8 K 2024-11-17T01:51:49,382 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/7528cef67934485a9743052ccf4ff7aa, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/130a4066353c46b7a7351041ca724c4c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/ae46911fdac34497ae101874d1b83677] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=36.5 K 2024-11-17T01:51:49,383 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0163584b9f64614b1fa2ab4ce39fb2d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731808305808 2024-11-17T01:51:49,383 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 7528cef67934485a9743052ccf4ff7aa, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731808305808 2024-11-17T01:51:49,383 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 130a4066353c46b7a7351041ca724c4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731808306629 2024-11-17T01:51:49,383 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8a1d3f7549447c2bf52d445730e6712, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731808306629 2024-11-17T01:51:49,384 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd8b09283d704b3791deccbbc2bc2612, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731808307283 2024-11-17T01:51:49,384 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting ae46911fdac34497ae101874d1b83677, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731808307283 2024-11-17T01:51:49,395 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:49,395 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/7d3bb56ebbd1422a82da9ae775572ba7 is 50, key is test_row_0/A:col10/1731808307283/Put/seqid=0 2024-11-17T01:51:49,397 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#B#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:49,398 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/79cd81e5051d4a56bbfe51b3f1c2a4c4 is 50, key is test_row_0/B:col10/1731808307283/Put/seqid=0 2024-11-17T01:51:49,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741898_1074 (size=12983) 2024-11-17T01:51:49,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741897_1073 (size=12983) 2024-11-17T01:51:49,421 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/79cd81e5051d4a56bbfe51b3f1c2a4c4 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/79cd81e5051d4a56bbfe51b3f1c2a4c4 2024-11-17T01:51:49,430 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/B of 5d8a196529c6f40b7321f17bb79ff0a8 into 79cd81e5051d4a56bbfe51b3f1c2a4c4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:49,430 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:49,430 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/B, priority=13, startTime=1731808309378; duration=0sec 2024-11-17T01:51:49,430 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:51:49,430 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:B 2024-11-17T01:51:49,430 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:51:49,431 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:51:49,431 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/C is initiating minor compaction (all files) 2024-11-17T01:51:49,431 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/C in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:49,432 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/fd8273e4810348d3bb84bd3bffaacc96, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/40f3c7302def4ec1b12687de81233114, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/409ad773707d4631b63772e14ea846a0] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=36.5 K 2024-11-17T01:51:49,432 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting fd8273e4810348d3bb84bd3bffaacc96, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731808305808 2024-11-17T01:51:49,432 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 40f3c7302def4ec1b12687de81233114, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731808306629 2024-11-17T01:51:49,433 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 409ad773707d4631b63772e14ea846a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731808307283 2024-11-17T01:51:49,442 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#C#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:49,442 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/cf1e26a08af94ee7b6127a8e56d913e9 is 50, key is test_row_0/C:col10/1731808307283/Put/seqid=0 2024-11-17T01:51:49,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741899_1075 (size=12983) 2024-11-17T01:51:49,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T01:51:49,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:49,470 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T01:51:49,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:49,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:49,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:49,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:49,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:49,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T01:51:49,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/8124afad5dd84e81919bbc4639fd627d is 50, key is test_row_0/A:col10/1731808308467/Put/seqid=0 2024-11-17T01:51:49,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741900_1076 (size=12301) 2024-11-17T01:51:49,508 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/8124afad5dd84e81919bbc4639fd627d 2024-11-17T01:51:49,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/0f288ad725084ff59266c73644577689 is 50, key is test_row_0/B:col10/1731808308467/Put/seqid=0 2024-11-17T01:51:49,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741901_1077 (size=12301) 2024-11-17T01:51:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:49,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:49,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808369631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:49,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:49,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808369633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:49,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:49,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808369634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:49,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:49,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808369634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:49,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:49,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808369635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:49,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:49,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808369745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:49,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808369745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:49,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808369746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:49,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808369746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:49,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:49,814 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/7d3bb56ebbd1422a82da9ae775572ba7 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/7d3bb56ebbd1422a82da9ae775572ba7 2024-11-17T01:51:49,821 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/A of 5d8a196529c6f40b7321f17bb79ff0a8 into 7d3bb56ebbd1422a82da9ae775572ba7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:49,821 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:49,821 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/A, priority=13, startTime=1731808309378; duration=0sec 2024-11-17T01:51:49,822 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:49,822 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:A 2024-11-17T01:51:49,855 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/cf1e26a08af94ee7b6127a8e56d913e9 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/cf1e26a08af94ee7b6127a8e56d913e9 2024-11-17T01:51:49,862 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/C of 5d8a196529c6f40b7321f17bb79ff0a8 into cf1e26a08af94ee7b6127a8e56d913e9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:49,863 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:49,863 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/C, priority=13, startTime=1731808309379; duration=0sec 2024-11-17T01:51:49,863 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:49,863 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:C 2024-11-17T01:51:49,926 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/0f288ad725084ff59266c73644577689 2024-11-17T01:51:49,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/66c20cb68d7e49efa1b23138b98adcce is 50, key is test_row_0/C:col10/1731808308467/Put/seqid=0 2024-11-17T01:51:49,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741902_1078 (size=12301) 2024-11-17T01:51:49,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808369964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:49,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808369964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:49,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808369964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808369964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:49,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T01:51:50,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:50,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:50,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808370274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:50,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808370274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:50,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:50,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:50,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808370275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:50,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808370275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:50,344 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/66c20cb68d7e49efa1b23138b98adcce 2024-11-17T01:51:50,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/8124afad5dd84e81919bbc4639fd627d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/8124afad5dd84e81919bbc4639fd627d 2024-11-17T01:51:50,357 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/8124afad5dd84e81919bbc4639fd627d, entries=150, sequenceid=318, filesize=12.0 K 2024-11-17T01:51:50,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/0f288ad725084ff59266c73644577689 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/0f288ad725084ff59266c73644577689 2024-11-17T01:51:50,375 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/0f288ad725084ff59266c73644577689, entries=150, sequenceid=318, filesize=12.0 K 2024-11-17T01:51:50,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/66c20cb68d7e49efa1b23138b98adcce as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/66c20cb68d7e49efa1b23138b98adcce 2024-11-17T01:51:50,384 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/66c20cb68d7e49efa1b23138b98adcce, entries=150, sequenceid=318, filesize=12.0 K 2024-11-17T01:51:50,385 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5d8a196529c6f40b7321f17bb79ff0a8 in 915ms, sequenceid=318, compaction requested=false 2024-11-17T01:51:50,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:50,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:50,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-17T01:51:50,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-17T01:51:50,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-17T01:51:50,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5390 sec 2024-11-17T01:51:50,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.5450 sec 2024-11-17T01:51:50,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:50,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T01:51:50,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:50,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:50,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:50,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:50,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:50,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:50,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/28c33a8e999c46d296d1faf81498375f is 50, key is test_row_0/A:col10/1731808310655/Put/seqid=0 2024-11-17T01:51:50,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741903_1079 (size=14741) 2024-11-17T01:51:50,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/28c33a8e999c46d296d1faf81498375f 2024-11-17T01:51:50,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/258d39212f4744c7a73cb3573ebc5986 is 50, key is test_row_0/B:col10/1731808310655/Put/seqid=0 2024-11-17T01:51:50,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741904_1080 (size=12301) 2024-11-17T01:51:50,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/258d39212f4744c7a73cb3573ebc5986 2024-11-17T01:51:50,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/4f8647a20109403d8cd8b7b8dfae8511 is 50, key is test_row_0/C:col10/1731808310655/Put/seqid=0 2024-11-17T01:51:50,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808370776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:50,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741905_1081 (size=12301) 2024-11-17T01:51:50,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/4f8647a20109403d8cd8b7b8dfae8511 2024-11-17T01:51:50,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808370784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:50,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:50,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:50,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:50,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808370784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:50,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808370785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:50,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808370785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:50,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:50,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:50,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/28c33a8e999c46d296d1faf81498375f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/28c33a8e999c46d296d1faf81498375f 2024-11-17T01:51:50,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/28c33a8e999c46d296d1faf81498375f, entries=200, sequenceid=333, filesize=14.4 K 2024-11-17T01:51:50,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/258d39212f4744c7a73cb3573ebc5986 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/258d39212f4744c7a73cb3573ebc5986 2024-11-17T01:51:50,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/258d39212f4744c7a73cb3573ebc5986, entries=150, sequenceid=333, filesize=12.0 K 2024-11-17T01:51:50,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/4f8647a20109403d8cd8b7b8dfae8511 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/4f8647a20109403d8cd8b7b8dfae8511 2024-11-17T01:51:50,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/4f8647a20109403d8cd8b7b8dfae8511, entries=150, sequenceid=333, filesize=12.0 K 2024-11-17T01:51:50,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5d8a196529c6f40b7321f17bb79ff0a8 in 175ms, sequenceid=333, compaction requested=true 2024-11-17T01:51:50,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:50,833 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:51:50,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:51:50,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:50,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:51:50,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:51:50,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:51:50,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T01:51:50,835 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:51:50,836 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:51:50,836 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/A is initiating minor compaction (all files) 2024-11-17T01:51:50,836 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/A in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:50,836 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/7d3bb56ebbd1422a82da9ae775572ba7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/8124afad5dd84e81919bbc4639fd627d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/28c33a8e999c46d296d1faf81498375f] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=39.1 K 2024-11-17T01:51:50,837 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d3bb56ebbd1422a82da9ae775572ba7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731808307283 2024-11-17T01:51:50,837 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:51:50,837 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/B is initiating minor compaction (all files) 2024-11-17T01:51:50,837 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/B in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:50,837 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/79cd81e5051d4a56bbfe51b3f1c2a4c4, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/0f288ad725084ff59266c73644577689, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/258d39212f4744c7a73cb3573ebc5986] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=36.7 K 2024-11-17T01:51:50,838 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8124afad5dd84e81919bbc4639fd627d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731808308467 2024-11-17T01:51:50,838 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 79cd81e5051d4a56bbfe51b3f1c2a4c4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731808307283 2024-11-17T01:51:50,839 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28c33a8e999c46d296d1faf81498375f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731808309624 2024-11-17T01:51:50,839 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f288ad725084ff59266c73644577689, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731808308467 2024-11-17T01:51:50,839 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 258d39212f4744c7a73cb3573ebc5986, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731808309629 2024-11-17T01:51:50,856 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#A#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:50,856 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/36a0920ea0b345ec85868f7d1f18f20e is 50, key is test_row_0/A:col10/1731808310655/Put/seqid=0 2024-11-17T01:51:50,856 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:50,857 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/370a6984998d40908c2ec1d7a2dab7a1 is 50, key is test_row_0/B:col10/1731808310655/Put/seqid=0 2024-11-17T01:51:50,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741906_1082 (size=13085) 2024-11-17T01:51:50,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741907_1083 (size=13085) 2024-11-17T01:51:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:50,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T01:51:50,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:50,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:50,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:50,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:50,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:50,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:50,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/97d5182d1f3d49d48d352fac50e8eb49 is 50, key is test_row_0/A:col10/1731808310763/Put/seqid=0 2024-11-17T01:51:50,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741908_1084 (size=14741) 2024-11-17T01:51:50,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/97d5182d1f3d49d48d352fac50e8eb49 2024-11-17T01:51:50,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/5feb54cead674234b63a0ce81a19ada3 is 50, key is test_row_0/B:col10/1731808310763/Put/seqid=0 2024-11-17T01:51:50,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741909_1085 (size=12301) 2024-11-17T01:51:50,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/5feb54cead674234b63a0ce81a19ada3 2024-11-17T01:51:50,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/dffce6327e7749e3b834e96a461adbee is 50, key is test_row_0/C:col10/1731808310763/Put/seqid=0 2024-11-17T01:51:50,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741910_1086 (size=12301) 2024-11-17T01:51:50,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808370940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T01:51:50,994 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:51:50,994 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:51:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-17T01:51:50,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T01:51:50,997 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:51:50,998 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:51:50,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:51:51,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808371054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:51,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T01:51:51,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T01:51:51,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:51,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:51,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:51,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:51,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:51,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808371264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:51,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:51,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:51,280 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/36a0920ea0b345ec85868f7d1f18f20e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/36a0920ea0b345ec85868f7d1f18f20e 2024-11-17T01:51:51,280 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/370a6984998d40908c2ec1d7a2dab7a1 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/370a6984998d40908c2ec1d7a2dab7a1 2024-11-17T01:51:51,289 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/A of 5d8a196529c6f40b7321f17bb79ff0a8 into 36a0920ea0b345ec85868f7d1f18f20e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:51,289 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:51,289 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/A, priority=13, startTime=1731808310833; duration=0sec 2024-11-17T01:51:51,289 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:51:51,289 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:A 2024-11-17T01:51:51,289 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:51:51,291 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/B of 5d8a196529c6f40b7321f17bb79ff0a8 into 370a6984998d40908c2ec1d7a2dab7a1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:51,291 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:51,291 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/B, priority=13, startTime=1731808310834; duration=0sec 2024-11-17T01:51:51,291 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:51,291 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:B 2024-11-17T01:51:51,292 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:51:51,292 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/C is initiating minor compaction (all files) 2024-11-17T01:51:51,292 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/C in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:51,292 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/cf1e26a08af94ee7b6127a8e56d913e9, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/66c20cb68d7e49efa1b23138b98adcce, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/4f8647a20109403d8cd8b7b8dfae8511] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=36.7 K 2024-11-17T01:51:51,293 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf1e26a08af94ee7b6127a8e56d913e9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731808307283 2024-11-17T01:51:51,293 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66c20cb68d7e49efa1b23138b98adcce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731808308467 2024-11-17T01:51:51,294 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f8647a20109403d8cd8b7b8dfae8511, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731808309629 2024-11-17T01:51:51,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T01:51:51,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:51,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:51,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T01:51:51,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:51,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:51,320 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#C#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:51,321 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/3fc0e95a2cd64fec9455d5a585f9cbf4 is 50, key is test_row_0/C:col10/1731808310655/Put/seqid=0 2024-11-17T01:51:51,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741911_1087 (size=13085) 2024-11-17T01:51:51,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/dffce6327e7749e3b834e96a461adbee 2024-11-17T01:51:51,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/97d5182d1f3d49d48d352fac50e8eb49 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/97d5182d1f3d49d48d352fac50e8eb49 2024-11-17T01:51:51,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/97d5182d1f3d49d48d352fac50e8eb49, entries=200, sequenceid=353, filesize=14.4 K 2024-11-17T01:51:51,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/5feb54cead674234b63a0ce81a19ada3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/5feb54cead674234b63a0ce81a19ada3 2024-11-17T01:51:51,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/5feb54cead674234b63a0ce81a19ada3, entries=150, sequenceid=353, filesize=12.0 K 2024-11-17T01:51:51,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/dffce6327e7749e3b834e96a461adbee as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/dffce6327e7749e3b834e96a461adbee 2024-11-17T01:51:51,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/dffce6327e7749e3b834e96a461adbee, entries=150, sequenceid=353, filesize=12.0 K 2024-11-17T01:51:51,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5d8a196529c6f40b7321f17bb79ff0a8 in 483ms, sequenceid=353, compaction requested=false 2024-11-17T01:51:51,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:51,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T01:51:51,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:51,470 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T01:51:51,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:51,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:51,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:51,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:51,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:51,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:51,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/1d76e6d1926348eea68f32ddb4ffee1f is 50, key is test_row_0/A:col10/1731808310901/Put/seqid=0 2024-11-17T01:51:51,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741912_1088 (size=12301) 2024-11-17T01:51:51,483 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/1d76e6d1926348eea68f32ddb4ffee1f 2024-11-17T01:51:51,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/f9f49d62575543d6a0577df90f9e09d8 is 50, key is test_row_0/B:col10/1731808310901/Put/seqid=0 2024-11-17T01:51:51,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741913_1089 (size=12301) 2024-11-17T01:51:51,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T01:51:51,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808371671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:51,732 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/3fc0e95a2cd64fec9455d5a585f9cbf4 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/3fc0e95a2cd64fec9455d5a585f9cbf4 2024-11-17T01:51:51,738 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/C of 5d8a196529c6f40b7321f17bb79ff0a8 into 3fc0e95a2cd64fec9455d5a585f9cbf4(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:51,738 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:51,738 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/C, priority=13, startTime=1731808310835; duration=0sec 2024-11-17T01:51:51,739 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:51,739 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:C 2024-11-17T01:51:51,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808371784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:51,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808371794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:51,801 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in 5d8a196529c6f40b7321f17bb79ff0a8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:51:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:51,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808371804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:51,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808371805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:51,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808371805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:51,810 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in 5d8a196529c6f40b7321f17bb79ff0a8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:51:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:51,810 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in 5d8a196529c6f40b7321f17bb79ff0a8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:51:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:51,810 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in 5d8a196529c6f40b7321f17bb79ff0a8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:51:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:51,914 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/f9f49d62575543d6a0577df90f9e09d8 2024-11-17T01:51:51,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/c72c917f8f4c448986ac85e63548f6ef is 50, key is test_row_0/C:col10/1731808310901/Put/seqid=0 2024-11-17T01:51:51,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741914_1090 (size=12301) 2024-11-17T01:51:52,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808371994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:52,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:52,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:52,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T01:51:52,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808372314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:52,327 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/c72c917f8f4c448986ac85e63548f6ef 2024-11-17T01:51:52,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/1d76e6d1926348eea68f32ddb4ffee1f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/1d76e6d1926348eea68f32ddb4ffee1f 2024-11-17T01:51:52,339 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/1d76e6d1926348eea68f32ddb4ffee1f, entries=150, sequenceid=367, filesize=12.0 K 2024-11-17T01:51:52,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/f9f49d62575543d6a0577df90f9e09d8 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/f9f49d62575543d6a0577df90f9e09d8 2024-11-17T01:51:52,346 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/f9f49d62575543d6a0577df90f9e09d8, entries=150, sequenceid=367, filesize=12.0 K 2024-11-17T01:51:52,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/c72c917f8f4c448986ac85e63548f6ef as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/c72c917f8f4c448986ac85e63548f6ef 2024-11-17T01:51:52,353 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/c72c917f8f4c448986ac85e63548f6ef, entries=150, sequenceid=367, filesize=12.0 K 2024-11-17T01:51:52,354 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5d8a196529c6f40b7321f17bb79ff0a8 in 884ms, sequenceid=367, compaction requested=true 2024-11-17T01:51:52,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:52,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:52,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-17T01:51:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-17T01:51:52,359 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-17T01:51:52,359 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3580 sec 2024-11-17T01:51:52,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.3650 sec 2024-11-17T01:51:52,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808311610, max timestamp=1731808312825Num uniques 859; 2024-11-17T01:51:52,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808311610, max timestamp=1731808312825Num uniques 859; 2024-11-17T01:51:52,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808311610, max timestamp=1731808312825Num uniques 859; 2024-11-17T01:51:52,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:51:52,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:51:52,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:51:52,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:52,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-17T01:51:52,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:52,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:51:52,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:52,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:51:52,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:52,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:51:52,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/59cee660194a4c17980fd8861821901c is 50, key is test_row_0/A:col10/1731808311610/Put/seqid=0 2024-11-17T01:51:52,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741915_1091 (size=14741) 2024-11-17T01:51:52,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808312936, max timestamp=1731808312965Num uniques 859; 2024-11-17T01:51:52,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808312936, max timestamp=1731808312965Num uniques 859; 2024-11-17T01:51:52,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808312936, max timestamp=1731808312965Num uniques 859; 2024-11-17T01:51:52,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:51:52,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:51:52,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:51:53,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808373071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:53,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:53,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:53,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T01:51:53,145 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:51:53,145 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:51:53,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-17T01:51:53,151 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:51:53,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T01:51:53,152 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:51:53,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:51:53,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808373184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:53,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:53,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:53,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:53,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T01:51:53,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/59cee660194a4c17980fd8861821901c 2024-11-17T01:51:53,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/8bec1fc46e7341f9b45092f407bb30f0 is 50, key is test_row_0/B:col10/1731808311610/Put/seqid=0 2024-11-17T01:51:53,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741916_1092 (size=12301) 2024-11-17T01:51:53,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T01:51:53,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:53,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:53,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:53,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:53,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:53,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808373394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:53,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T01:51:53,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:53,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:53,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:53,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:53,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:53,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:53,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T01:51:53,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T01:51:53,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:53,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:53,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:53,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:53,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:53,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:53,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/8bec1fc46e7341f9b45092f407bb30f0 2024-11-17T01:51:53,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/20f42abc50614c08a620326399e0e7fa is 50, key is test_row_0/C:col10/1731808311610/Put/seqid=0 2024-11-17T01:51:53,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:53,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808373707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:53,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:53,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:53,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:53,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741917_1093 (size=12301) 2024-11-17T01:51:53,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T01:51:53,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:53,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:53,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:53,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:53,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T01:51:53,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808373814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:53,824 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in 5d8a196529c6f40b7321f17bb79ff0a8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:51:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:53,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808373834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:53,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808373835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:53,843 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in 5d8a196529c6f40b7321f17bb79ff0a8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:51:53,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808373835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:53,844 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in 5d8a196529c6f40b7321f17bb79ff0a8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:51:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:53,845 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in 5d8a196529c6f40b7321f17bb79ff0a8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:51:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:53,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T01:51:53,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:53,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:53,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:53,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:53,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:54,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T01:51:54,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:54,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:54,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:54,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:54,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:54,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/20f42abc50614c08a620326399e0e7fa 2024-11-17T01:51:54,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/59cee660194a4c17980fd8861821901c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/59cee660194a4c17980fd8861821901c 2024-11-17T01:51:54,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/59cee660194a4c17980fd8861821901c, entries=200, sequenceid=393, filesize=14.4 K 2024-11-17T01:51:54,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/8bec1fc46e7341f9b45092f407bb30f0 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/8bec1fc46e7341f9b45092f407bb30f0 2024-11-17T01:51:54,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/8bec1fc46e7341f9b45092f407bb30f0, entries=150, sequenceid=393, filesize=12.0 K 2024-11-17T01:51:54,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/20f42abc50614c08a620326399e0e7fa as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/20f42abc50614c08a620326399e0e7fa 2024-11-17T01:51:54,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/20f42abc50614c08a620326399e0e7fa, entries=150, sequenceid=393, filesize=12.0 K 2024-11-17T01:51:54,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for 5d8a196529c6f40b7321f17bb79ff0a8 in 1311ms, sequenceid=393, compaction requested=true 2024-11-17T01:51:54,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:54,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:51:54,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:54,161 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:51:54,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:51:54,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:51:54,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:51:54,162 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:51:54,163 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54868 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:51:54,163 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/A is initiating minor compaction (all files) 2024-11-17T01:51:54,163 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/A in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:54,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T01:51:54,163 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/36a0920ea0b345ec85868f7d1f18f20e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/97d5182d1f3d49d48d352fac50e8eb49, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/1d76e6d1926348eea68f32ddb4ffee1f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/59cee660194a4c17980fd8861821901c] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=53.6 K 2024-11-17T01:51:54,163 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 36a0920ea0b345ec85868f7d1f18f20e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731808309629 2024-11-17T01:51:54,164 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:51:54,164 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/B is initiating minor compaction (all files) 2024-11-17T01:51:54,164 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/B in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:54,164 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/370a6984998d40908c2ec1d7a2dab7a1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/5feb54cead674234b63a0ce81a19ada3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/f9f49d62575543d6a0577df90f9e09d8, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/8bec1fc46e7341f9b45092f407bb30f0] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=48.8 K 2024-11-17T01:51:54,164 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 97d5182d1f3d49d48d352fac50e8eb49, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731808310757 2024-11-17T01:51:54,165 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 370a6984998d40908c2ec1d7a2dab7a1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731808309629 2024-11-17T01:51:54,165 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d76e6d1926348eea68f32ddb4ffee1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731808310901 2024-11-17T01:51:54,165 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5feb54cead674234b63a0ce81a19ada3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731808310763 2024-11-17T01:51:54,165 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 59cee660194a4c17980fd8861821901c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731808311610 2024-11-17T01:51:54,166 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9f49d62575543d6a0577df90f9e09d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731808310901 2024-11-17T01:51:54,166 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bec1fc46e7341f9b45092f407bb30f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731808311610 2024-11-17T01:51:54,186 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#A#compaction#81 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:54,186 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/07a68b43382f4bea8c8a50f0d67da9f9 is 50, key is test_row_0/A:col10/1731808311610/Put/seqid=0 2024-11-17T01:51:54,190 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#B#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:54,190 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/676d62681aa64274912b3ece1ee9d433 is 50, key is test_row_0/B:col10/1731808311610/Put/seqid=0 2024-11-17T01:51:54,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741918_1094 (size=13221) 2024-11-17T01:51:54,207 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/07a68b43382f4bea8c8a50f0d67da9f9 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/07a68b43382f4bea8c8a50f0d67da9f9 2024-11-17T01:51:54,217 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/A of 5d8a196529c6f40b7321f17bb79ff0a8 into 07a68b43382f4bea8c8a50f0d67da9f9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:54,217 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:54,217 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/A, priority=12, startTime=1731808314161; duration=0sec 2024-11-17T01:51:54,218 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:51:54,218 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:A 2024-11-17T01:51:54,218 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:51:54,221 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:51:54,221 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/C is initiating minor compaction (all files) 2024-11-17T01:51:54,222 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/C in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:54,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741919_1095 (size=13221) 2024-11-17T01:51:54,222 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/3fc0e95a2cd64fec9455d5a585f9cbf4, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/dffce6327e7749e3b834e96a461adbee, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/c72c917f8f4c448986ac85e63548f6ef, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/20f42abc50614c08a620326399e0e7fa] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=48.8 K 2024-11-17T01:51:54,225 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fc0e95a2cd64fec9455d5a585f9cbf4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731808309629 2024-11-17T01:51:54,226 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting dffce6327e7749e3b834e96a461adbee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731808310763 2024-11-17T01:51:54,226 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting c72c917f8f4c448986ac85e63548f6ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731808310901 2024-11-17T01:51:54,227 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 20f42abc50614c08a620326399e0e7fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731808311610 2024-11-17T01:51:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:54,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-17T01:51:54,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:54,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:51:54,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:54,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:51:54,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:54,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:51:54,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T01:51:54,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:54,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:54,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:54,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:54,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:54,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/340005c423a14c32b8b27d2b9ef00def is 50, key is test_row_0/A:col10/1731808313066/Put/seqid=0 2024-11-17T01:51:54,269 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#C#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:54,269 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/d6fe0c6a1ba24489a318847dc11cbbb2 is 50, key is test_row_0/C:col10/1731808311610/Put/seqid=0 2024-11-17T01:51:54,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808374267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:54,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:54,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:54,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:54,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741920_1096 (size=14741) 2024-11-17T01:51:54,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/340005c423a14c32b8b27d2b9ef00def 2024-11-17T01:51:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T01:51:54,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741921_1097 (size=13221) 2024-11-17T01:51:54,311 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/d6fe0c6a1ba24489a318847dc11cbbb2 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/d6fe0c6a1ba24489a318847dc11cbbb2 2024-11-17T01:51:54,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/275d4dd0ce344e7897d739bbe1a54ac9 is 50, key is test_row_0/B:col10/1731808313066/Put/seqid=0 2024-11-17T01:51:54,319 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/C of 5d8a196529c6f40b7321f17bb79ff0a8 into d6fe0c6a1ba24489a318847dc11cbbb2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:54,319 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:54,319 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/C, priority=12, startTime=1731808314161; duration=0sec 2024-11-17T01:51:54,319 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:54,319 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:C 2024-11-17T01:51:54,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741922_1098 (size=12301) 2024-11-17T01:51:54,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808374384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:54,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:54,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:54,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:54,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T01:51:54,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:54,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:54,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:54,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:54,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:54,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T01:51:54,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:54,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:54,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:54,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:54,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:54,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808374604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:54,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:54,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:54,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:54,629 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/676d62681aa64274912b3ece1ee9d433 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/676d62681aa64274912b3ece1ee9d433 2024-11-17T01:51:54,638 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/B of 5d8a196529c6f40b7321f17bb79ff0a8 into 676d62681aa64274912b3ece1ee9d433(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:54,638 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:54,638 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/B, priority=12, startTime=1731808314161; duration=0sec 2024-11-17T01:51:54,638 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:54,638 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:B 2024-11-17T01:51:54,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T01:51:54,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:54,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:54,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:54,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:54,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:54,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/275d4dd0ce344e7897d739bbe1a54ac9 2024-11-17T01:51:54,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/5c69b9e244874d55a34ce553d5483ac0 is 50, key is test_row_0/C:col10/1731808313066/Put/seqid=0 2024-11-17T01:51:54,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741923_1099 (size=12301) 2024-11-17T01:51:54,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T01:51:54,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:54,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:54,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:54,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:54,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:54,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808374914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:55,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T01:51:55,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:55,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:55,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:55,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:55,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:55,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/5c69b9e244874d55a34ce553d5483ac0 2024-11-17T01:51:55,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/340005c423a14c32b8b27d2b9ef00def as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/340005c423a14c32b8b27d2b9ef00def 2024-11-17T01:51:55,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/340005c423a14c32b8b27d2b9ef00def, entries=200, sequenceid=431, filesize=14.4 K 2024-11-17T01:51:55,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/275d4dd0ce344e7897d739bbe1a54ac9 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/275d4dd0ce344e7897d739bbe1a54ac9 2024-11-17T01:51:55,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T01:51:55,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:55,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:55,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:55,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:55,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:55,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/275d4dd0ce344e7897d739bbe1a54ac9, entries=150, sequenceid=431, filesize=12.0 K 2024-11-17T01:51:55,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/5c69b9e244874d55a34ce553d5483ac0 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/5c69b9e244874d55a34ce553d5483ac0 2024-11-17T01:51:55,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/5c69b9e244874d55a34ce553d5483ac0, entries=150, sequenceid=431, filesize=12.0 K 2024-11-17T01:51:55,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 5d8a196529c6f40b7321f17bb79ff0a8 in 946ms, sequenceid=431, compaction requested=false 2024-11-17T01:51:55,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T01:51:55,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T01:51:55,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:55,316 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-17T01:51:55,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:55,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:55,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:55,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:55,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:55,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:55,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/7ff89c36242d46b5b84020d5cb05af71 is 50, key is test_row_0/A:col10/1731808314265/Put/seqid=0 2024-11-17T01:51:55,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741924_1100 (size=12301) 2024-11-17T01:51:55,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:55,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808315486, max timestamp=1731808315507Num uniques 859; 2024-11-17T01:51:55,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808315486, max timestamp=1731808315507Num uniques 859; 2024-11-17T01:51:55,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:51:55,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:51:55,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808315486, max timestamp=1731808315507Num uniques 859; 2024-11-17T01:51:55,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:51:55,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808375627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:55,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:55,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:55,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:55,736 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/7ff89c36242d46b5b84020d5cb05af71 2024-11-17T01:51:55,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808375744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:55,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/32614199d7544113921197018f189445 is 50, key is test_row_0/B:col10/1731808314265/Put/seqid=0 2024-11-17T01:51:55,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741925_1101 (size=12301) 2024-11-17T01:51:55,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808375954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:56,179 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/32614199d7544113921197018f189445 2024-11-17T01:51:56,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/ca01d199bcad4e6fb7f246e47d896144 is 50, key is test_row_0/C:col10/1731808314265/Put/seqid=0 2024-11-17T01:51:56,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741926_1102 (size=12301) 2024-11-17T01:51:56,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808376264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:56,599 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/ca01d199bcad4e6fb7f246e47d896144 2024-11-17T01:51:56,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/7ff89c36242d46b5b84020d5cb05af71 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/7ff89c36242d46b5b84020d5cb05af71 2024-11-17T01:51:56,611 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/7ff89c36242d46b5b84020d5cb05af71, entries=150, sequenceid=442, filesize=12.0 K 2024-11-17T01:51:56,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/32614199d7544113921197018f189445 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/32614199d7544113921197018f189445 2024-11-17T01:51:56,621 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/32614199d7544113921197018f189445, entries=150, sequenceid=442, filesize=12.0 K 2024-11-17T01:51:56,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/ca01d199bcad4e6fb7f246e47d896144 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/ca01d199bcad4e6fb7f246e47d896144 2024-11-17T01:51:56,628 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/ca01d199bcad4e6fb7f246e47d896144, entries=150, sequenceid=442, filesize=12.0 K 2024-11-17T01:51:56,629 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 5d8a196529c6f40b7321f17bb79ff0a8 in 1313ms, sequenceid=442, compaction requested=true 2024-11-17T01:51:56,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:56,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:56,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-17T01:51:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-17T01:51:56,633 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-17T01:51:56,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4790 sec 2024-11-17T01:51:56,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 3.4900 sec 2024-11-17T01:51:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:56,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-17T01:51:56,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:56,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T01:51:56,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:56,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T01:51:56,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:56,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T01:51:56,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/ff2a08db6ab44c6698d56737f0448278 is 50, key is test_row_0/A:col10/1731808316784/Put/seqid=0 2024-11-17T01:51:56,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741927_1103 (size=14741) 2024-11-17T01:51:56,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808376820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:56,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808376935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:56,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:56,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:56,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:57,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808377144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:57,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/ff2a08db6ab44c6698d56737f0448278 2024-11-17T01:51:57,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/beb05d6c8e634654bb5c2ad12be1f30e is 50, key is test_row_0/B:col10/1731808316784/Put/seqid=0 2024-11-17T01:51:57,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741928_1104 (size=12301) 2024-11-17T01:51:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T01:51:57,325 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:51:57,325 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:51:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-17T01:51:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T01:51:57,327 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:51:57,328 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:51:57,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:51:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T01:51:57,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808377464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:57,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T01:51:57,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:57,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:57,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:57,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:57,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:57,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/beb05d6c8e634654bb5c2ad12be1f30e 2024-11-17T01:51:57,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/ec91bcb2b1984c648bc8e175acad75f8 is 50, key is test_row_0/C:col10/1731808316784/Put/seqid=0 2024-11-17T01:51:57,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741929_1105 (size=12301) 2024-11-17T01:51:57,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T01:51:57,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:57,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:57,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:57,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:57,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T01:51:57,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T01:51:57,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:57,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:57,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:57,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:57,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:57,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:57,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48320 deadline: 1731808377854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:57,858 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in 5d8a196529c6f40b7321f17bb79ff0a8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:51:57,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:57,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:57,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:57,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:57,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48304 deadline: 1731808377874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:57,878 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in 5d8a196529c6f40b7321f17bb79ff0a8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:51:57,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:57,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:57,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:57,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48282 deadline: 1731808377884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:57,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48316 deadline: 1731808377884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:57,892 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in 5d8a196529c6f40b7321f17bb79ff0a8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:51:57,892 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in 5d8a196529c6f40b7321f17bb79ff0a8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:51:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:57,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:57,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T01:51:57,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:57,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:57,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:57,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:57,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:57,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:57,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T01:51:57,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:57,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808377974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:57,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:57,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:57,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:58,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/ec91bcb2b1984c648bc8e175acad75f8 2024-11-17T01:51:58,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/ff2a08db6ab44c6698d56737f0448278 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/ff2a08db6ab44c6698d56737f0448278 2024-11-17T01:51:58,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/ff2a08db6ab44c6698d56737f0448278, entries=200, sequenceid=480, filesize=14.4 K 2024-11-17T01:51:58,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/beb05d6c8e634654bb5c2ad12be1f30e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/beb05d6c8e634654bb5c2ad12be1f30e 2024-11-17T01:51:58,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/beb05d6c8e634654bb5c2ad12be1f30e, entries=150, sequenceid=480, filesize=12.0 K 2024-11-17T01:51:58,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/ec91bcb2b1984c648bc8e175acad75f8 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/ec91bcb2b1984c648bc8e175acad75f8 2024-11-17T01:51:58,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/ec91bcb2b1984c648bc8e175acad75f8, entries=150, sequenceid=480, filesize=12.0 K 2024-11-17T01:51:58,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 5d8a196529c6f40b7321f17bb79ff0a8 in 1279ms, sequenceid=480, compaction requested=true 2024-11-17T01:51:58,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:58,065 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:51:58,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:51:58,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:58,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:51:58,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:58,066 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:51:58,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:51:58,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:51:58,068 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:51:58,068 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/A is initiating minor compaction (all files) 2024-11-17T01:51:58,068 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/A in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:58,068 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/07a68b43382f4bea8c8a50f0d67da9f9, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/340005c423a14c32b8b27d2b9ef00def, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/7ff89c36242d46b5b84020d5cb05af71, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/ff2a08db6ab44c6698d56737f0448278] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=53.7 K 2024-11-17T01:51:58,068 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:51:58,068 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/B is initiating minor compaction (all files) 2024-11-17T01:51:58,068 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/B in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:58,069 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/676d62681aa64274912b3ece1ee9d433, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/275d4dd0ce344e7897d739bbe1a54ac9, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/32614199d7544113921197018f189445, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/beb05d6c8e634654bb5c2ad12be1f30e] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=48.9 K 2024-11-17T01:51:58,069 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07a68b43382f4bea8c8a50f0d67da9f9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731808311610 2024-11-17T01:51:58,069 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 676d62681aa64274912b3ece1ee9d433, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731808311610 2024-11-17T01:51:58,070 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 340005c423a14c32b8b27d2b9ef00def, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731808313044 2024-11-17T01:51:58,070 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 275d4dd0ce344e7897d739bbe1a54ac9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731808313044 2024-11-17T01:51:58,070 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ff89c36242d46b5b84020d5cb05af71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731808314235 2024-11-17T01:51:58,071 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 32614199d7544113921197018f189445, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731808314235 2024-11-17T01:51:58,071 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff2a08db6ab44c6698d56737f0448278, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731808315583 2024-11-17T01:51:58,071 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting beb05d6c8e634654bb5c2ad12be1f30e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731808315583 2024-11-17T01:51:58,087 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#A#compaction#93 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:58,088 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/713c3c0da36046ddb9c1b1fa73fac311 is 50, key is test_row_0/A:col10/1731808316784/Put/seqid=0 2024-11-17T01:51:58,092 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#B#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:58,093 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/7c7178503089487f9374a6653fa0efc6 is 50, key is test_row_0/B:col10/1731808316784/Put/seqid=0 2024-11-17T01:51:58,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T01:51:58,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:58,094 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-17T01:51:58,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:58,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:58,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:58,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:58,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:58,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:58,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741930_1106 (size=13357) 2024-11-17T01:51:58,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/4ccb97fb9518409f9d97a942b8140540 is 50, key is test_row_0/A:col10/1731808316819/Put/seqid=0 2024-11-17T01:51:58,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741931_1107 (size=13357) 2024-11-17T01:51:58,119 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/713c3c0da36046ddb9c1b1fa73fac311 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/713c3c0da36046ddb9c1b1fa73fac311 2024-11-17T01:51:58,134 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/7c7178503089487f9374a6653fa0efc6 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/7c7178503089487f9374a6653fa0efc6 2024-11-17T01:51:58,139 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/A of 5d8a196529c6f40b7321f17bb79ff0a8 into 713c3c0da36046ddb9c1b1fa73fac311(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:58,139 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:58,139 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/A, priority=12, startTime=1731808318065; duration=0sec 2024-11-17T01:51:58,139 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:51:58,139 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:A 2024-11-17T01:51:58,139 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:51:58,144 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:51:58,144 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/C is initiating minor compaction (all files) 2024-11-17T01:51:58,144 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/C in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:58,144 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/d6fe0c6a1ba24489a318847dc11cbbb2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/5c69b9e244874d55a34ce553d5483ac0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/ca01d199bcad4e6fb7f246e47d896144, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/ec91bcb2b1984c648bc8e175acad75f8] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=48.9 K 2024-11-17T01:51:58,145 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6fe0c6a1ba24489a318847dc11cbbb2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731808311610 2024-11-17T01:51:58,147 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c69b9e244874d55a34ce553d5483ac0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731808313044 2024-11-17T01:51:58,148 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca01d199bcad4e6fb7f246e47d896144, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731808314235 2024-11-17T01:51:58,148 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/B of 5d8a196529c6f40b7321f17bb79ff0a8 into 7c7178503089487f9374a6653fa0efc6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:58,149 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:58,149 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec91bcb2b1984c648bc8e175acad75f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731808315583 2024-11-17T01:51:58,149 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/B, priority=12, startTime=1731808318066; duration=0sec 2024-11-17T01:51:58,149 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:58,149 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:B 2024-11-17T01:51:58,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741932_1108 (size=12301) 2024-11-17T01:51:58,154 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/4ccb97fb9518409f9d97a942b8140540 2024-11-17T01:51:58,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/d3d097a8f19d44be8a0d42cf84aa12da is 50, key is test_row_0/B:col10/1731808316819/Put/seqid=0 2024-11-17T01:51:58,167 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#C#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:51:58,168 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/623329d08ae74f08a4a497dfd102bc22 is 50, key is test_row_0/C:col10/1731808316784/Put/seqid=0 2024-11-17T01:51:58,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741933_1109 (size=12301) 2024-11-17T01:51:58,176 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/d3d097a8f19d44be8a0d42cf84aa12da 2024-11-17T01:51:58,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741934_1110 (size=13357) 2024-11-17T01:51:58,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/6e46b8a7c6af492f915a7b0e95fa77f6 is 50, key is test_row_0/C:col10/1731808316819/Put/seqid=0 2024-11-17T01:51:58,196 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/623329d08ae74f08a4a497dfd102bc22 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/623329d08ae74f08a4a497dfd102bc22 2024-11-17T01:51:58,205 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/C of 5d8a196529c6f40b7321f17bb79ff0a8 into 623329d08ae74f08a4a497dfd102bc22(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:51:58,205 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:58,206 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/C, priority=12, startTime=1731808318066; duration=0sec 2024-11-17T01:51:58,206 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:51:58,206 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:C 2024-11-17T01:51:58,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741935_1111 (size=12301) 2024-11-17T01:51:58,238 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/6e46b8a7c6af492f915a7b0e95fa77f6 2024-11-17T01:51:58,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/4ccb97fb9518409f9d97a942b8140540 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/4ccb97fb9518409f9d97a942b8140540 2024-11-17T01:51:58,265 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/4ccb97fb9518409f9d97a942b8140540, entries=150, sequenceid=488, filesize=12.0 K 2024-11-17T01:51:58,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/d3d097a8f19d44be8a0d42cf84aa12da as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/d3d097a8f19d44be8a0d42cf84aa12da 2024-11-17T01:51:58,276 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/d3d097a8f19d44be8a0d42cf84aa12da, entries=150, sequenceid=488, filesize=12.0 K 2024-11-17T01:51:58,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/6e46b8a7c6af492f915a7b0e95fa77f6 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/6e46b8a7c6af492f915a7b0e95fa77f6 2024-11-17T01:51:58,287 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/6e46b8a7c6af492f915a7b0e95fa77f6, entries=150, sequenceid=488, filesize=12.0 K 2024-11-17T01:51:58,288 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5d8a196529c6f40b7321f17bb79ff0a8 in 194ms, sequenceid=488, compaction requested=false 2024-11-17T01:51:58,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:58,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:58,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-17T01:51:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-17T01:51:58,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-17T01:51:58,293 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 962 msec 2024-11-17T01:51:58,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 970 msec 2024-11-17T01:51:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T01:51:58,465 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:51:58,465 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:51:58,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-17T01:51:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T01:51:58,468 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:51:58,469 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:51:58,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:51:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T01:51:58,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T01:51:58,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:58,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:58,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:58,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-17T01:51:58,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-17T01:51:58,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-17T01:51:58,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-17T01:51:58,627 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 160 msec 2024-11-17T01:51:58,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T01:51:58,784 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:51:58,785 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:51:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-17T01:51:58,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T01:51:58,787 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:51:58,788 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:51:58,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:51:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T01:51:58,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T01:51:58,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:58,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:51:58,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:58,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-17T01:51:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-17T01:51:58,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-17T01:51:58,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 157 msec 2024-11-17T01:51:58,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 163 msec 2024-11-17T01:51:59,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:51:59,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T01:51:59,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:51:59,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:59,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:51:59,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:59,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:51:59,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:51:59,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/a90ea20efd3c4853a72cab34196d2b01 is 50, key is test_row_0/A:col10/1731808319001/Put/seqid=0 2024-11-17T01:51:59,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741936_1112 (size=12301) 2024-11-17T01:51:59,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808319034, max timestamp=1731808319090Num uniques 859; 2024-11-17T01:51:59,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808319034, max timestamp=1731808319090Num uniques 859; 2024-11-17T01:51:59,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808319034, max timestamp=1731808319090Num uniques 859; 2024-11-17T01:51:59,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:51:59,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:51:59,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:51:59,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T01:51:59,105 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:51:59,106 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:51:59,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-17T01:51:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T01:51:59,110 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:51:59,111 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:51:59,111 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:51:59,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:59,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808379172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:59,203 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-17T01:51:59,203 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:51:59,204 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-17T01:51:59,204 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:51:59,207 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-17T01:51:59,207 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:51:59,208 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:51:59,208 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:51:59,208 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:51:59,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:51:59,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:51:59,209 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:51:59,209 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:51:59,209 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:51:59,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:51:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T01:51:59,215 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-17T01:51:59,215 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:51:59,215 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:51:59,215 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:51:59,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:51:59,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T01:51:59,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:59,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:59,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:59,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:59,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:59,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808379284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:59,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T01:51:59,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:59,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:59,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:59,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:59,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:59,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T01:51:59,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/a90ea20efd3c4853a72cab34196d2b01 2024-11-17T01:51:59,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/30d2c56f132d434ba8c664630f39acc1 is 50, key is test_row_0/B:col10/1731808319001/Put/seqid=0 2024-11-17T01:51:59,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741937_1113 (size=12301) 2024-11-17T01:51:59,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808379494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:59,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T01:51:59,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:59,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:59,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:59,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:59,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:59,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T01:51:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:59,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:59,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:59,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:59,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T01:51:59,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:51:59,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48308 deadline: 1731808379804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:51:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:51:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:51:59,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/30d2c56f132d434ba8c664630f39acc1 2024-11-17T01:51:59,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/8776e1c4c9ba44afa7372c73bd164213 is 50, key is test_row_0/C:col10/1731808319001/Put/seqid=0 2024-11-17T01:51:59,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741938_1114 (size=12301) 2024-11-17T01:51:59,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T01:51:59,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:59,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:51:59,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:51:59,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:59,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:51:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T01:52:00,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:00,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:52:00,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:00,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T01:52:00,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:00,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:52:00,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:00,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T01:52:00,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/8776e1c4c9ba44afa7372c73bd164213 2024-11-17T01:52:00,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/a90ea20efd3c4853a72cab34196d2b01 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/a90ea20efd3c4853a72cab34196d2b01 2024-11-17T01:52:00,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/a90ea20efd3c4853a72cab34196d2b01, entries=150, sequenceid=503, filesize=12.0 K 2024-11-17T01:52:00,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/30d2c56f132d434ba8c664630f39acc1 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/30d2c56f132d434ba8c664630f39acc1 2024-11-17T01:52:00,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/30d2c56f132d434ba8c664630f39acc1, entries=150, sequenceid=503, filesize=12.0 K 2024-11-17T01:52:00,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/8776e1c4c9ba44afa7372c73bd164213 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/8776e1c4c9ba44afa7372c73bd164213 2024-11-17T01:52:00,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/8776e1c4c9ba44afa7372c73bd164213, entries=150, sequenceid=503, filesize=12.0 K 2024-11-17T01:52:00,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 5d8a196529c6f40b7321f17bb79ff0a8 in 1278ms, sequenceid=503, compaction requested=true 2024-11-17T01:52:00,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:52:00,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:52:00,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:00,286 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:00,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:52:00,286 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:00,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:00,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d8a196529c6f40b7321f17bb79ff0a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:52:00,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:00,287 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:00,287 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:00,287 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/A is initiating minor compaction (all files) 2024-11-17T01:52:00,287 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/B is initiating minor compaction (all files) 2024-11-17T01:52:00,287 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/A in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:00,287 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/B in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:00,287 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/713c3c0da36046ddb9c1b1fa73fac311, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/4ccb97fb9518409f9d97a942b8140540, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/a90ea20efd3c4853a72cab34196d2b01] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=37.1 K 2024-11-17T01:52:00,287 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/7c7178503089487f9374a6653fa0efc6, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/d3d097a8f19d44be8a0d42cf84aa12da, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/30d2c56f132d434ba8c664630f39acc1] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=37.1 K 2024-11-17T01:52:00,288 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 713c3c0da36046ddb9c1b1fa73fac311, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731808315583 2024-11-17T01:52:00,288 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c7178503089487f9374a6653fa0efc6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731808315583 2024-11-17T01:52:00,288 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ccb97fb9518409f9d97a942b8140540, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731808316796 2024-11-17T01:52:00,288 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting d3d097a8f19d44be8a0d42cf84aa12da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731808316796 2024-11-17T01:52:00,288 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting a90ea20efd3c4853a72cab34196d2b01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1731808318999 2024-11-17T01:52:00,288 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 30d2c56f132d434ba8c664630f39acc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1731808318999 2024-11-17T01:52:00,296 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#B#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:00,297 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#A#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:00,297 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/0ada607ebbb649c48688f83da20beb92 is 50, key is test_row_0/B:col10/1731808319001/Put/seqid=0 2024-11-17T01:52:00,297 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/d7750d2e735940c98b77ed6b4ce5f2d6 is 50, key is test_row_0/A:col10/1731808319001/Put/seqid=0 2024-11-17T01:52:00,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741939_1115 (size=13459) 2024-11-17T01:52:00,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741940_1116 (size=13459) 2024-11-17T01:52:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:52:00,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-17T01:52:00,316 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-17T01:52:00,316 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:52:00,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:52:00,316 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:00,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:52:00,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:52:00,316 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:00,316 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:52:00,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:52:00,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:52:00,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:52:00,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/daa46846b7554b3bbf96bedff1b836a3 is 50, key is test_row_0/A:col10/1731808319142/Put/seqid=0 2024-11-17T01:52:00,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741941_1117 (size=12301) 2024-11-17T01:52:00,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T01:52:00,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:00,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:52:00,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:00,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T01:52:00,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:00,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:52:00,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:00,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,502 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T01:52:00,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T01:52:00,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:00,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:52:00,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:00,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,712 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/0ada607ebbb649c48688f83da20beb92 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/0ada607ebbb649c48688f83da20beb92 2024-11-17T01:52:00,712 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/d7750d2e735940c98b77ed6b4ce5f2d6 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/d7750d2e735940c98b77ed6b4ce5f2d6 2024-11-17T01:52:00,717 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/A of 5d8a196529c6f40b7321f17bb79ff0a8 into d7750d2e735940c98b77ed6b4ce5f2d6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:00,717 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/B of 5d8a196529c6f40b7321f17bb79ff0a8 into 0ada607ebbb649c48688f83da20beb92(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:00,717 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:52:00,717 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:52:00,717 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/A, priority=13, startTime=1731808320286; duration=0sec 2024-11-17T01:52:00,717 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/B, priority=13, startTime=1731808320286; duration=0sec 2024-11-17T01:52:00,718 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:00,718 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:B 2024-11-17T01:52:00,718 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:00,718 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:00,718 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:A 2024-11-17T01:52:00,719 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:00,719 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5d8a196529c6f40b7321f17bb79ff0a8/C is initiating minor compaction (all files) 2024-11-17T01:52:00,719 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d8a196529c6f40b7321f17bb79ff0a8/C in TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:00,719 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/623329d08ae74f08a4a497dfd102bc22, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/6e46b8a7c6af492f915a7b0e95fa77f6, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/8776e1c4c9ba44afa7372c73bd164213] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp, totalSize=37.1 K 2024-11-17T01:52:00,719 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 623329d08ae74f08a4a497dfd102bc22, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731808315583 2024-11-17T01:52:00,720 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e46b8a7c6af492f915a7b0e95fa77f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731808316796 2024-11-17T01:52:00,720 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 8776e1c4c9ba44afa7372c73bd164213, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1731808318999 2024-11-17T01:52:00,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/daa46846b7554b3bbf96bedff1b836a3 2024-11-17T01:52:00,728 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d8a196529c6f40b7321f17bb79ff0a8#C#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:00,729 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/f814b777a6e949b481f44073514b394e is 50, key is test_row_0/C:col10/1731808319001/Put/seqid=0 2024-11-17T01:52:00,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741942_1118 (size=13459) 2024-11-17T01:52:00,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/b8ed6e79d97c4d1bb904892b73920bbb is 50, key is test_row_0/B:col10/1731808319142/Put/seqid=0 2024-11-17T01:52:00,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741943_1119 (size=12301) 2024-11-17T01:52:00,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T01:52:00,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:00,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:52:00,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:00,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T01:52:00,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:00,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:52:00,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:00,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:01,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T01:52:01,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:01,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:52:01,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:01,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:01,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:01,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/b8ed6e79d97c4d1bb904892b73920bbb 2024-11-17T01:52:01,140 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/f814b777a6e949b481f44073514b394e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/f814b777a6e949b481f44073514b394e 2024-11-17T01:52:01,147 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d8a196529c6f40b7321f17bb79ff0a8/C of 5d8a196529c6f40b7321f17bb79ff0a8 into f814b777a6e949b481f44073514b394e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:01,147 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:52:01,147 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., storeName=5d8a196529c6f40b7321f17bb79ff0a8/C, priority=13, startTime=1731808320286; duration=0sec 2024-11-17T01:52:01,147 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:01,147 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d8a196529c6f40b7321f17bb79ff0a8:C 2024-11-17T01:52:01,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/54f4ae2b287e43b5b1741c5ce43753f5 is 50, key is test_row_0/C:col10/1731808319142/Put/seqid=0 2024-11-17T01:52:01,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741944_1120 (size=12301) 2024-11-17T01:52:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T01:52:01,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T01:52:01,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:01,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:52:01,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:01,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:01,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:01,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T01:52:01,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:01,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:52:01,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:01,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:01,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:01,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/54f4ae2b287e43b5b1741c5ce43753f5 2024-11-17T01:52:01,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T01:52:01,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:01,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. as already flushing 2024-11-17T01:52:01,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:01,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:01,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:01,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/daa46846b7554b3bbf96bedff1b836a3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/daa46846b7554b3bbf96bedff1b836a3 2024-11-17T01:52:01,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/daa46846b7554b3bbf96bedff1b836a3, entries=150, sequenceid=540, filesize=12.0 K 2024-11-17T01:52:01,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/b8ed6e79d97c4d1bb904892b73920bbb as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/b8ed6e79d97c4d1bb904892b73920bbb 2024-11-17T01:52:01,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/b8ed6e79d97c4d1bb904892b73920bbb, entries=150, sequenceid=540, filesize=12.0 K 2024-11-17T01:52:01,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/54f4ae2b287e43b5b1741c5ce43753f5 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/54f4ae2b287e43b5b1741c5ce43753f5 2024-11-17T01:52:01,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/54f4ae2b287e43b5b1741c5ce43753f5, entries=150, sequenceid=540, filesize=12.0 K 2024-11-17T01:52:01,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=0 B/0 for 5d8a196529c6f40b7321f17bb79ff0a8 in 1301ms, sequenceid=540, compaction requested=false 2024-11-17T01:52:01,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:52:01,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T01:52:01,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:01,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 5d8a196529c6f40b7321f17bb79ff0a8: 2024-11-17T01:52:01,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:01,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-17T01:52:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-17T01:52:01,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-17T01:52:01,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6340 sec 2024-11-17T01:52:01,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 2.6420 sec 2024-11-17T01:52:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T01:52:03,285 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:07,916 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-17T01:52:07,916 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:52:07,916 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:07,916 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:07,916 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:52:07,946 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-17T01:52:07,946 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:52:07,946 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:07,946 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:07,946 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:52:07,956 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-17T01:52:07,956 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:52:07,956 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:07,956 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:07,956 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:52:07,997 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-17T01:52:07,997 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:52:07,997 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:07,997 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:07,997 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:52:07,997 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T01:52:07,997 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-17T01:52:07,997 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-17T01:52:07,997 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 211 2024-11-17T01:52:07,997 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-17T01:52:07,997 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-17T01:52:07,997 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T01:52:07,997 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2649 2024-11-17T01:52:07,997 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2655 2024-11-17T01:52:07,997 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T01:52:07,997 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1167 2024-11-17T01:52:07,998 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3497 rows 2024-11-17T01:52:07,998 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1171 2024-11-17T01:52:07,998 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3509 rows 2024-11-17T01:52:07,998 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T01:52:07,998 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:52:07,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:07,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:07,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:52:08,002 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T01:52:08,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T01:52:08,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T01:52:08,010 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808328009"}]},"ts":"1731808328009"} 2024-11-17T01:52:08,012 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T01:52:08,012 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T01:52:08,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T01:52:08,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d8a196529c6f40b7321f17bb79ff0a8, UNASSIGN}] 2024-11-17T01:52:08,022 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d8a196529c6f40b7321f17bb79ff0a8, UNASSIGN 2024-11-17T01:52:08,023 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=5d8a196529c6f40b7321f17bb79ff0a8, regionState=CLOSING, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:08,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d8a196529c6f40b7321f17bb79ff0a8, UNASSIGN because future has completed 2024-11-17T01:52:08,026 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T01:52:08,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173}] 2024-11-17T01:52:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T01:52:08,183 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:52:08,183 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T01:52:08,184 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 5d8a196529c6f40b7321f17bb79ff0a8, disabling compactions & flushes 2024-11-17T01:52:08,184 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:08,184 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:08,184 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. after waiting 0 ms 2024-11-17T01:52:08,184 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:08,184 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 5d8a196529c6f40b7321f17bb79ff0a8 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-17T01:52:08,184 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=A 2024-11-17T01:52:08,184 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:08,184 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=B 2024-11-17T01:52:08,184 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:08,184 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d8a196529c6f40b7321f17bb79ff0a8, store=C 2024-11-17T01:52:08,185 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:08,189 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/00ce3781d65c44a2adaf9bfec32186cf is 50, key is test_row_0/A:col10/1731808327995/Put/seqid=0 2024-11-17T01:52:08,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741945_1121 (size=9857) 2024-11-17T01:52:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T01:52:08,594 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=551 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/00ce3781d65c44a2adaf9bfec32186cf 2024-11-17T01:52:08,601 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/7d6f300761fc4e6c84ab45ceb1844e62 is 50, key is test_row_0/B:col10/1731808327995/Put/seqid=0 2024-11-17T01:52:08,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741946_1122 (size=9857) 2024-11-17T01:52:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T01:52:09,006 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=551 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/7d6f300761fc4e6c84ab45ceb1844e62 2024-11-17T01:52:09,014 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/fff2bb7d9dea46d5baf0d59d2167341c is 50, key is test_row_0/C:col10/1731808327995/Put/seqid=0 2024-11-17T01:52:09,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741947_1123 (size=9857) 2024-11-17T01:52:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T01:52:09,419 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=551 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/fff2bb7d9dea46d5baf0d59d2167341c 2024-11-17T01:52:09,429 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/A/00ce3781d65c44a2adaf9bfec32186cf as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/00ce3781d65c44a2adaf9bfec32186cf 2024-11-17T01:52:09,435 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/00ce3781d65c44a2adaf9bfec32186cf, entries=100, sequenceid=551, filesize=9.6 K 2024-11-17T01:52:09,435 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/B/7d6f300761fc4e6c84ab45ceb1844e62 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/7d6f300761fc4e6c84ab45ceb1844e62 2024-11-17T01:52:09,440 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/7d6f300761fc4e6c84ab45ceb1844e62, entries=100, sequenceid=551, filesize=9.6 K 2024-11-17T01:52:09,441 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/.tmp/C/fff2bb7d9dea46d5baf0d59d2167341c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/fff2bb7d9dea46d5baf0d59d2167341c 2024-11-17T01:52:09,446 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/fff2bb7d9dea46d5baf0d59d2167341c, entries=100, sequenceid=551, filesize=9.6 K 2024-11-17T01:52:09,447 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 5d8a196529c6f40b7321f17bb79ff0a8 in 1263ms, sequenceid=551, compaction requested=true 2024-11-17T01:52:09,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/4245f6acd1114582b93c091b842c9d93, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/3d4c69b722e545f3819de3de50ee34a7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/2db4017dffcf40cfaebe5d24b4f9ac73, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/b4db29f6534d43c5a44b79f48494d8c8, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/aba3b24898ca40e7bc8bbfbfbe1e956e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/d8bcdd47aad743a3a07fcb7be36d59ed, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/9496a12c522c44b9941e1a47ac0b56fd, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/81cf77005e464700b2300a7b5dd6df18, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/c8640e1b126c41799fd65afe73f76154, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/a0d60219b8c7466981d465acae7a8015, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/8a8fa05a3bd94235afb63b9e64c8ea23, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/b9df8a84627849f6b468e695fad70f00, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/0461162c235148abbe730e2eafecba60, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/1052f0f05bc24a7d8a8e6e33cd884880, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/ed0ea1ae2aa24348bd194981e7a181cd, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/4c2d9d6124d941128d5dfc801e6d2eed, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/ae583273dd82464087f178ce35e4aa42, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/f0163584b9f64614b1fa2ab4ce39fb2d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/d8a1d3f7549447c2bf52d445730e6712, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/cd8b09283d704b3791deccbbc2bc2612, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/7d3bb56ebbd1422a82da9ae775572ba7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/8124afad5dd84e81919bbc4639fd627d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/28c33a8e999c46d296d1faf81498375f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/36a0920ea0b345ec85868f7d1f18f20e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/97d5182d1f3d49d48d352fac50e8eb49, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/1d76e6d1926348eea68f32ddb4ffee1f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/59cee660194a4c17980fd8861821901c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/07a68b43382f4bea8c8a50f0d67da9f9, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/340005c423a14c32b8b27d2b9ef00def, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/7ff89c36242d46b5b84020d5cb05af71, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/ff2a08db6ab44c6698d56737f0448278, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/713c3c0da36046ddb9c1b1fa73fac311, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/4ccb97fb9518409f9d97a942b8140540, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/a90ea20efd3c4853a72cab34196d2b01] to archive 2024-11-17T01:52:09,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T01:52:09,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/4245f6acd1114582b93c091b842c9d93 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/4245f6acd1114582b93c091b842c9d93 2024-11-17T01:52:09,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/3d4c69b722e545f3819de3de50ee34a7 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/3d4c69b722e545f3819de3de50ee34a7 2024-11-17T01:52:09,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/2db4017dffcf40cfaebe5d24b4f9ac73 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/2db4017dffcf40cfaebe5d24b4f9ac73 2024-11-17T01:52:09,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/b4db29f6534d43c5a44b79f48494d8c8 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/b4db29f6534d43c5a44b79f48494d8c8 2024-11-17T01:52:09,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/aba3b24898ca40e7bc8bbfbfbe1e956e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/aba3b24898ca40e7bc8bbfbfbe1e956e 2024-11-17T01:52:09,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/d8bcdd47aad743a3a07fcb7be36d59ed to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/d8bcdd47aad743a3a07fcb7be36d59ed 2024-11-17T01:52:09,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/9496a12c522c44b9941e1a47ac0b56fd to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/9496a12c522c44b9941e1a47ac0b56fd 2024-11-17T01:52:09,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/81cf77005e464700b2300a7b5dd6df18 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/81cf77005e464700b2300a7b5dd6df18 2024-11-17T01:52:09,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/c8640e1b126c41799fd65afe73f76154 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/c8640e1b126c41799fd65afe73f76154 2024-11-17T01:52:09,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/a0d60219b8c7466981d465acae7a8015 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/a0d60219b8c7466981d465acae7a8015 2024-11-17T01:52:09,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/8a8fa05a3bd94235afb63b9e64c8ea23 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/8a8fa05a3bd94235afb63b9e64c8ea23 2024-11-17T01:52:09,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/b9df8a84627849f6b468e695fad70f00 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/b9df8a84627849f6b468e695fad70f00 2024-11-17T01:52:09,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/0461162c235148abbe730e2eafecba60 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/0461162c235148abbe730e2eafecba60 2024-11-17T01:52:09,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/1052f0f05bc24a7d8a8e6e33cd884880 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/1052f0f05bc24a7d8a8e6e33cd884880 2024-11-17T01:52:09,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/ed0ea1ae2aa24348bd194981e7a181cd to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/ed0ea1ae2aa24348bd194981e7a181cd 2024-11-17T01:52:09,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/4c2d9d6124d941128d5dfc801e6d2eed to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/4c2d9d6124d941128d5dfc801e6d2eed 2024-11-17T01:52:09,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/ae583273dd82464087f178ce35e4aa42 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/ae583273dd82464087f178ce35e4aa42 2024-11-17T01:52:09,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/f0163584b9f64614b1fa2ab4ce39fb2d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/f0163584b9f64614b1fa2ab4ce39fb2d 2024-11-17T01:52:09,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/d8a1d3f7549447c2bf52d445730e6712 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/d8a1d3f7549447c2bf52d445730e6712 2024-11-17T01:52:09,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/cd8b09283d704b3791deccbbc2bc2612 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/cd8b09283d704b3791deccbbc2bc2612 2024-11-17T01:52:09,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/7d3bb56ebbd1422a82da9ae775572ba7 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/7d3bb56ebbd1422a82da9ae775572ba7 2024-11-17T01:52:09,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/8124afad5dd84e81919bbc4639fd627d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/8124afad5dd84e81919bbc4639fd627d 2024-11-17T01:52:09,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/28c33a8e999c46d296d1faf81498375f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/28c33a8e999c46d296d1faf81498375f 2024-11-17T01:52:09,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/36a0920ea0b345ec85868f7d1f18f20e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/36a0920ea0b345ec85868f7d1f18f20e 2024-11-17T01:52:09,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/97d5182d1f3d49d48d352fac50e8eb49 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/97d5182d1f3d49d48d352fac50e8eb49 2024-11-17T01:52:09,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/1d76e6d1926348eea68f32ddb4ffee1f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/1d76e6d1926348eea68f32ddb4ffee1f 2024-11-17T01:52:09,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/59cee660194a4c17980fd8861821901c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/59cee660194a4c17980fd8861821901c 2024-11-17T01:52:09,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/07a68b43382f4bea8c8a50f0d67da9f9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/07a68b43382f4bea8c8a50f0d67da9f9 2024-11-17T01:52:09,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/340005c423a14c32b8b27d2b9ef00def to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/340005c423a14c32b8b27d2b9ef00def 2024-11-17T01:52:09,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/7ff89c36242d46b5b84020d5cb05af71 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/7ff89c36242d46b5b84020d5cb05af71 2024-11-17T01:52:09,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/ff2a08db6ab44c6698d56737f0448278 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/ff2a08db6ab44c6698d56737f0448278 2024-11-17T01:52:09,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/713c3c0da36046ddb9c1b1fa73fac311 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/713c3c0da36046ddb9c1b1fa73fac311 2024-11-17T01:52:09,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/4ccb97fb9518409f9d97a942b8140540 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/4ccb97fb9518409f9d97a942b8140540 2024-11-17T01:52:09,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/a90ea20efd3c4853a72cab34196d2b01 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/a90ea20efd3c4853a72cab34196d2b01 2024-11-17T01:52:09,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/6281bf62b1e84a67b688052abf7ce33a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/3adc00a0b0f84b7bb6aa1a7272c90e18, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/b308fdbfb044454e836770dad70478a2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/c5bf1fee5ef84ad39421bc71b87d7071, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/53a3dbf3769d425a93eeac7019195f5f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/084be17988404df0b2aa721b4395ca4d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/d530213fbf794de494bdf1a12dc60e84, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/9d64877f27ca4949bd31786ea3245dc4, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/5aff1b7e6d9a47e9858b77a40e885ef8, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/3f78dbb7652f47d2a275c6266594eed0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/8456ecc9dca34c4e840bebf18e8ace7e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/2ee60e53af9041de8f6aeb19033ec359, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/d6e2058286fb47d2968faac70f2821e7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/b6971f7259fb49daa379b0e79dedffa3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/ab99064ee7734320897342da85446e0a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/3c33cd4b1c764c29962076c230a8ba99, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/7528cef67934485a9743052ccf4ff7aa, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/45637b972ad1406e90c9aee1842f5937, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/130a4066353c46b7a7351041ca724c4c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/79cd81e5051d4a56bbfe51b3f1c2a4c4, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/ae46911fdac34497ae101874d1b83677, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/0f288ad725084ff59266c73644577689, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/370a6984998d40908c2ec1d7a2dab7a1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/258d39212f4744c7a73cb3573ebc5986, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/5feb54cead674234b63a0ce81a19ada3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/f9f49d62575543d6a0577df90f9e09d8, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/676d62681aa64274912b3ece1ee9d433, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/8bec1fc46e7341f9b45092f407bb30f0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/275d4dd0ce344e7897d739bbe1a54ac9, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/32614199d7544113921197018f189445, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/7c7178503089487f9374a6653fa0efc6, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/beb05d6c8e634654bb5c2ad12be1f30e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/d3d097a8f19d44be8a0d42cf84aa12da, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/30d2c56f132d434ba8c664630f39acc1] to archive 2024-11-17T01:52:09,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T01:52:09,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/6281bf62b1e84a67b688052abf7ce33a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/6281bf62b1e84a67b688052abf7ce33a 2024-11-17T01:52:09,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/3adc00a0b0f84b7bb6aa1a7272c90e18 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/3adc00a0b0f84b7bb6aa1a7272c90e18 2024-11-17T01:52:09,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/b308fdbfb044454e836770dad70478a2 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/b308fdbfb044454e836770dad70478a2 2024-11-17T01:52:09,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/c5bf1fee5ef84ad39421bc71b87d7071 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/c5bf1fee5ef84ad39421bc71b87d7071 2024-11-17T01:52:09,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/53a3dbf3769d425a93eeac7019195f5f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/53a3dbf3769d425a93eeac7019195f5f 2024-11-17T01:52:09,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/084be17988404df0b2aa721b4395ca4d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/084be17988404df0b2aa721b4395ca4d 2024-11-17T01:52:09,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/d530213fbf794de494bdf1a12dc60e84 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/d530213fbf794de494bdf1a12dc60e84 2024-11-17T01:52:09,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/9d64877f27ca4949bd31786ea3245dc4 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/9d64877f27ca4949bd31786ea3245dc4 2024-11-17T01:52:09,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/5aff1b7e6d9a47e9858b77a40e885ef8 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/5aff1b7e6d9a47e9858b77a40e885ef8 2024-11-17T01:52:09,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/3f78dbb7652f47d2a275c6266594eed0 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/3f78dbb7652f47d2a275c6266594eed0 2024-11-17T01:52:09,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/8456ecc9dca34c4e840bebf18e8ace7e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/8456ecc9dca34c4e840bebf18e8ace7e 2024-11-17T01:52:09,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/2ee60e53af9041de8f6aeb19033ec359 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/2ee60e53af9041de8f6aeb19033ec359 2024-11-17T01:52:09,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/d6e2058286fb47d2968faac70f2821e7 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/d6e2058286fb47d2968faac70f2821e7 2024-11-17T01:52:09,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/b6971f7259fb49daa379b0e79dedffa3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/b6971f7259fb49daa379b0e79dedffa3 2024-11-17T01:52:09,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/ab99064ee7734320897342da85446e0a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/ab99064ee7734320897342da85446e0a 2024-11-17T01:52:09,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/3c33cd4b1c764c29962076c230a8ba99 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/3c33cd4b1c764c29962076c230a8ba99 2024-11-17T01:52:09,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/7528cef67934485a9743052ccf4ff7aa to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/7528cef67934485a9743052ccf4ff7aa 2024-11-17T01:52:09,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/45637b972ad1406e90c9aee1842f5937 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/45637b972ad1406e90c9aee1842f5937 2024-11-17T01:52:09,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/130a4066353c46b7a7351041ca724c4c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/130a4066353c46b7a7351041ca724c4c 2024-11-17T01:52:09,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/79cd81e5051d4a56bbfe51b3f1c2a4c4 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/79cd81e5051d4a56bbfe51b3f1c2a4c4 2024-11-17T01:52:09,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/ae46911fdac34497ae101874d1b83677 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/ae46911fdac34497ae101874d1b83677 2024-11-17T01:52:09,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/0f288ad725084ff59266c73644577689 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/0f288ad725084ff59266c73644577689 2024-11-17T01:52:09,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/370a6984998d40908c2ec1d7a2dab7a1 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/370a6984998d40908c2ec1d7a2dab7a1 2024-11-17T01:52:09,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/258d39212f4744c7a73cb3573ebc5986 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/258d39212f4744c7a73cb3573ebc5986 2024-11-17T01:52:09,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/5feb54cead674234b63a0ce81a19ada3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/5feb54cead674234b63a0ce81a19ada3 2024-11-17T01:52:09,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/f9f49d62575543d6a0577df90f9e09d8 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/f9f49d62575543d6a0577df90f9e09d8 2024-11-17T01:52:09,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/676d62681aa64274912b3ece1ee9d433 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/676d62681aa64274912b3ece1ee9d433 2024-11-17T01:52:09,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/8bec1fc46e7341f9b45092f407bb30f0 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/8bec1fc46e7341f9b45092f407bb30f0 2024-11-17T01:52:09,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/275d4dd0ce344e7897d739bbe1a54ac9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/275d4dd0ce344e7897d739bbe1a54ac9 2024-11-17T01:52:09,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/32614199d7544113921197018f189445 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/32614199d7544113921197018f189445 2024-11-17T01:52:09,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/7c7178503089487f9374a6653fa0efc6 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/7c7178503089487f9374a6653fa0efc6 2024-11-17T01:52:09,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/beb05d6c8e634654bb5c2ad12be1f30e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/beb05d6c8e634654bb5c2ad12be1f30e 2024-11-17T01:52:09,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/d3d097a8f19d44be8a0d42cf84aa12da to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/d3d097a8f19d44be8a0d42cf84aa12da 2024-11-17T01:52:09,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/30d2c56f132d434ba8c664630f39acc1 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/30d2c56f132d434ba8c664630f39acc1 2024-11-17T01:52:09,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/646e2588863d41888035ce3ecaf4423c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/f5d86bf0bbf341448e43f0e4698e15da, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/02d397dd1904478fac6b69acbc3c40cc, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/e613a3e7451d4a2a83765583fc949478, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/85e7010e894d409dbea9ca4df13b4148, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/1cd4ed6dba8d4e459172f2f8006c1c04, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/e5aff052a5f742c284d8fdac77b6ef07, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/ad7149306fee49ff9e3f660ab24400be, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/29004c73b687472bb2e826fddd121fd1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/847a767fd3e444e0855eeaa8e6e51ace, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/84c6c532d75e4cc99f7646817a6fdbef, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/9a9055b691a44c36b88c931c276e69fb, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/9ccafd59389148bb92a9fb1b05796d15, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/2b79946d6b974a08aeceb7ff6f167508, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/7147c661c7e34cc1952fabc49120d9c1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/b80738aeef2044ef8a74b5df84c89061, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/fd8273e4810348d3bb84bd3bffaacc96, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/924514e3957f4899bae2e3f14431ee6e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/40f3c7302def4ec1b12687de81233114, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/cf1e26a08af94ee7b6127a8e56d913e9, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/409ad773707d4631b63772e14ea846a0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/66c20cb68d7e49efa1b23138b98adcce, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/3fc0e95a2cd64fec9455d5a585f9cbf4, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/4f8647a20109403d8cd8b7b8dfae8511, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/dffce6327e7749e3b834e96a461adbee, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/c72c917f8f4c448986ac85e63548f6ef, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/d6fe0c6a1ba24489a318847dc11cbbb2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/20f42abc50614c08a620326399e0e7fa, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/5c69b9e244874d55a34ce553d5483ac0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/ca01d199bcad4e6fb7f246e47d896144, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/623329d08ae74f08a4a497dfd102bc22, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/ec91bcb2b1984c648bc8e175acad75f8, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/6e46b8a7c6af492f915a7b0e95fa77f6, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/8776e1c4c9ba44afa7372c73bd164213] to archive 2024-11-17T01:52:09,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T01:52:09,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/646e2588863d41888035ce3ecaf4423c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/646e2588863d41888035ce3ecaf4423c 2024-11-17T01:52:09,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/f5d86bf0bbf341448e43f0e4698e15da to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/f5d86bf0bbf341448e43f0e4698e15da 2024-11-17T01:52:09,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/02d397dd1904478fac6b69acbc3c40cc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/02d397dd1904478fac6b69acbc3c40cc 2024-11-17T01:52:09,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/e613a3e7451d4a2a83765583fc949478 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/e613a3e7451d4a2a83765583fc949478 2024-11-17T01:52:09,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/85e7010e894d409dbea9ca4df13b4148 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/85e7010e894d409dbea9ca4df13b4148 2024-11-17T01:52:09,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/1cd4ed6dba8d4e459172f2f8006c1c04 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/1cd4ed6dba8d4e459172f2f8006c1c04 2024-11-17T01:52:09,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/e5aff052a5f742c284d8fdac77b6ef07 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/e5aff052a5f742c284d8fdac77b6ef07 2024-11-17T01:52:09,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/ad7149306fee49ff9e3f660ab24400be to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/ad7149306fee49ff9e3f660ab24400be 2024-11-17T01:52:09,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/29004c73b687472bb2e826fddd121fd1 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/29004c73b687472bb2e826fddd121fd1 2024-11-17T01:52:09,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/847a767fd3e444e0855eeaa8e6e51ace to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/847a767fd3e444e0855eeaa8e6e51ace 2024-11-17T01:52:09,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/84c6c532d75e4cc99f7646817a6fdbef to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/84c6c532d75e4cc99f7646817a6fdbef 2024-11-17T01:52:09,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/9a9055b691a44c36b88c931c276e69fb to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/9a9055b691a44c36b88c931c276e69fb 2024-11-17T01:52:09,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/9ccafd59389148bb92a9fb1b05796d15 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/9ccafd59389148bb92a9fb1b05796d15 2024-11-17T01:52:09,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/2b79946d6b974a08aeceb7ff6f167508 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/2b79946d6b974a08aeceb7ff6f167508 2024-11-17T01:52:09,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/7147c661c7e34cc1952fabc49120d9c1 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/7147c661c7e34cc1952fabc49120d9c1 2024-11-17T01:52:09,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/b80738aeef2044ef8a74b5df84c89061 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/b80738aeef2044ef8a74b5df84c89061 2024-11-17T01:52:09,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/fd8273e4810348d3bb84bd3bffaacc96 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/fd8273e4810348d3bb84bd3bffaacc96 2024-11-17T01:52:09,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/924514e3957f4899bae2e3f14431ee6e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/924514e3957f4899bae2e3f14431ee6e 2024-11-17T01:52:09,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/40f3c7302def4ec1b12687de81233114 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/40f3c7302def4ec1b12687de81233114 2024-11-17T01:52:09,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/cf1e26a08af94ee7b6127a8e56d913e9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/cf1e26a08af94ee7b6127a8e56d913e9 2024-11-17T01:52:09,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/409ad773707d4631b63772e14ea846a0 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/409ad773707d4631b63772e14ea846a0 2024-11-17T01:52:09,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/66c20cb68d7e49efa1b23138b98adcce to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/66c20cb68d7e49efa1b23138b98adcce 2024-11-17T01:52:09,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/3fc0e95a2cd64fec9455d5a585f9cbf4 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/3fc0e95a2cd64fec9455d5a585f9cbf4 2024-11-17T01:52:09,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/4f8647a20109403d8cd8b7b8dfae8511 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/4f8647a20109403d8cd8b7b8dfae8511 2024-11-17T01:52:09,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/dffce6327e7749e3b834e96a461adbee to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/dffce6327e7749e3b834e96a461adbee 2024-11-17T01:52:09,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/c72c917f8f4c448986ac85e63548f6ef to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/c72c917f8f4c448986ac85e63548f6ef 2024-11-17T01:52:09,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/d6fe0c6a1ba24489a318847dc11cbbb2 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/d6fe0c6a1ba24489a318847dc11cbbb2 2024-11-17T01:52:09,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/20f42abc50614c08a620326399e0e7fa to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/20f42abc50614c08a620326399e0e7fa 2024-11-17T01:52:09,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/5c69b9e244874d55a34ce553d5483ac0 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/5c69b9e244874d55a34ce553d5483ac0 2024-11-17T01:52:09,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/ca01d199bcad4e6fb7f246e47d896144 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/ca01d199bcad4e6fb7f246e47d896144 2024-11-17T01:52:09,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/623329d08ae74f08a4a497dfd102bc22 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/623329d08ae74f08a4a497dfd102bc22 2024-11-17T01:52:09,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/ec91bcb2b1984c648bc8e175acad75f8 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/ec91bcb2b1984c648bc8e175acad75f8 2024-11-17T01:52:09,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/6e46b8a7c6af492f915a7b0e95fa77f6 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/6e46b8a7c6af492f915a7b0e95fa77f6 2024-11-17T01:52:09,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/8776e1c4c9ba44afa7372c73bd164213 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/8776e1c4c9ba44afa7372c73bd164213 2024-11-17T01:52:09,629 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/recovered.edits/554.seqid, newMaxSeqId=554, maxSeqId=1 2024-11-17T01:52:09,630 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. 2024-11-17T01:52:09,630 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 5d8a196529c6f40b7321f17bb79ff0a8: Waiting for close lock at 1731808328184Running coprocessor pre-close hooks at 1731808328184Disabling compacts and flushes for region at 1731808328184Disabling writes for close at 1731808328184Obtaining lock to block concurrent updates at 1731808328184Preparing flush snapshotting stores in 5d8a196529c6f40b7321f17bb79ff0a8 at 1731808328184Finished memstore snapshotting TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731808328185 (+1 ms)Flushing stores of TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8. at 1731808328185Flushing 5d8a196529c6f40b7321f17bb79ff0a8/A: creating writer at 1731808328185Flushing 5d8a196529c6f40b7321f17bb79ff0a8/A: appending metadata at 1731808328188 (+3 ms)Flushing 5d8a196529c6f40b7321f17bb79ff0a8/A: closing flushed file at 1731808328188Flushing 5d8a196529c6f40b7321f17bb79ff0a8/B: creating writer at 1731808328598 (+410 ms)Flushing 5d8a196529c6f40b7321f17bb79ff0a8/B: appending metadata at 1731808328601 (+3 ms)Flushing 5d8a196529c6f40b7321f17bb79ff0a8/B: closing flushed file at 1731808328601Flushing 5d8a196529c6f40b7321f17bb79ff0a8/C: creating writer at 1731808329011 (+410 ms)Flushing 5d8a196529c6f40b7321f17bb79ff0a8/C: appending metadata at 1731808329014 (+3 ms)Flushing 5d8a196529c6f40b7321f17bb79ff0a8/C: closing flushed file at 1731808329014Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@577635b2: reopening flushed file at 1731808329428 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e78b25c: reopening flushed file at 1731808329435 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fe4934: reopening flushed file at 1731808329440 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 5d8a196529c6f40b7321f17bb79ff0a8 in 1263ms, sequenceid=551, compaction requested=true at 1731808329447 (+7 ms)Writing region close event to WAL at 1731808329624 (+177 ms)Running coprocessor post-close hooks at 1731808329630 (+6 ms)Closed at 1731808329630 2024-11-17T01:52:09,633 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:52:09,634 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=5d8a196529c6f40b7321f17bb79ff0a8, regionState=CLOSED 2024-11-17T01:52:09,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 because future has completed 2024-11-17T01:52:09,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-17T01:52:09,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 5d8a196529c6f40b7321f17bb79ff0a8, server=c6c0d7a0a7c0,41157,1731808292173 in 1.6110 sec 2024-11-17T01:52:09,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-17T01:52:09,643 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d8a196529c6f40b7321f17bb79ff0a8, UNASSIGN in 1.6200 sec 2024-11-17T01:52:09,645 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-17T01:52:09,645 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6290 sec 2024-11-17T01:52:09,647 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808329647"}]},"ts":"1731808329647"} 2024-11-17T01:52:09,649 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T01:52:09,649 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T01:52:09,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6470 sec 2024-11-17T01:52:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T01:52:10,166 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:10,171 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T01:52:10,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:52:10,179 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:52:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-17T01:52:10,180 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:52:10,186 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:52:10,190 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A, FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B, FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C, FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/recovered.edits] 2024-11-17T01:52:10,193 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/00ce3781d65c44a2adaf9bfec32186cf to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/00ce3781d65c44a2adaf9bfec32186cf 2024-11-17T01:52:10,194 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/d7750d2e735940c98b77ed6b4ce5f2d6 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/d7750d2e735940c98b77ed6b4ce5f2d6 2024-11-17T01:52:10,196 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/daa46846b7554b3bbf96bedff1b836a3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/A/daa46846b7554b3bbf96bedff1b836a3 2024-11-17T01:52:10,198 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/0ada607ebbb649c48688f83da20beb92 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/0ada607ebbb649c48688f83da20beb92 2024-11-17T01:52:10,199 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/7d6f300761fc4e6c84ab45ceb1844e62 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/7d6f300761fc4e6c84ab45ceb1844e62 2024-11-17T01:52:10,200 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/b8ed6e79d97c4d1bb904892b73920bbb to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/B/b8ed6e79d97c4d1bb904892b73920bbb 2024-11-17T01:52:10,203 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/54f4ae2b287e43b5b1741c5ce43753f5 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/54f4ae2b287e43b5b1741c5ce43753f5 2024-11-17T01:52:10,204 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/f814b777a6e949b481f44073514b394e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/f814b777a6e949b481f44073514b394e 2024-11-17T01:52:10,205 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/fff2bb7d9dea46d5baf0d59d2167341c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/C/fff2bb7d9dea46d5baf0d59d2167341c 2024-11-17T01:52:10,208 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/recovered.edits/554.seqid to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8/recovered.edits/554.seqid 2024-11-17T01:52:10,208 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5d8a196529c6f40b7321f17bb79ff0a8 2024-11-17T01:52:10,208 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T01:52:10,213 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:52:10,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-17T01:52:10,222 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T01:52:10,251 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T01:52:10,253 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:52:10,253 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T01:52:10,253 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731808330253"}]},"ts":"9223372036854775807"} 2024-11-17T01:52:10,257 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T01:52:10,257 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5d8a196529c6f40b7321f17bb79ff0a8, NAME => 'TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T01:52:10,257 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T01:52:10,257 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731808330257"}]},"ts":"9223372036854775807"} 2024-11-17T01:52:10,260 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T01:52:10,260 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:52:10,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 88 msec 2024-11-17T01:52:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-17T01:52:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T01:52:10,286 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:10,298 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1864014360_22 at /127.0.0.1:37822 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=345 (was 224) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5094 (was 5572) 2024-11-17T01:52:10,307 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=345, ProcessCount=11, AvailableMemoryMB=5093 2024-11-17T01:52:10,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T01:52:10,310 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:52:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T01:52:10,312 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T01:52:10,312 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:10,312 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-17T01:52:10,313 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T01:52:10,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T01:52:10,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741948_1124 (size=1061) 2024-11-17T01:52:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T01:52:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T01:52:10,725 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1 2024-11-17T01:52:10,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741949_1125 (size=53) 2024-11-17T01:52:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T01:52:11,137 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:52:11,137 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 619e0c9a5ad3ede5c5b6d8d44658cb1a, disabling compactions & flushes 2024-11-17T01:52:11,137 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:11,137 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:11,137 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. after waiting 0 ms 2024-11-17T01:52:11,137 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:11,137 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:11,138 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: Waiting for close lock at 1731808331137Disabling compacts and flushes for region at 1731808331137Disabling writes for close at 1731808331137Writing region close event to WAL at 1731808331137Closed at 1731808331137 2024-11-17T01:52:11,140 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T01:52:11,141 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731808331140"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731808331140"}]},"ts":"1731808331140"} 2024-11-17T01:52:11,145 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T01:52:11,147 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T01:52:11,147 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808331147"}]},"ts":"1731808331147"} 2024-11-17T01:52:11,149 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T01:52:11,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619e0c9a5ad3ede5c5b6d8d44658cb1a, ASSIGN}] 2024-11-17T01:52:11,151 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619e0c9a5ad3ede5c5b6d8d44658cb1a, ASSIGN 2024-11-17T01:52:11,152 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=619e0c9a5ad3ede5c5b6d8d44658cb1a, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,41157,1731808292173; forceNewPlan=false, retain=false 2024-11-17T01:52:11,303 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=619e0c9a5ad3ede5c5b6d8d44658cb1a, regionState=OPENING, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:11,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=619e0c9a5ad3ede5c5b6d8d44658cb1a, ASSIGN because future has completed 2024-11-17T01:52:11,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173}] 2024-11-17T01:52:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T01:52:11,465 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:11,465 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:52:11,466 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:11,466 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:52:11,466 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:11,466 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:11,468 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:11,470 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:52:11,471 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 619e0c9a5ad3ede5c5b6d8d44658cb1a columnFamilyName A 2024-11-17T01:52:11,472 DEBUG [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:11,473 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] regionserver.HStore(327): Store=619e0c9a5ad3ede5c5b6d8d44658cb1a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:52:11,473 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:11,475 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:52:11,475 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 619e0c9a5ad3ede5c5b6d8d44658cb1a columnFamilyName B 2024-11-17T01:52:11,475 DEBUG [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:11,475 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] regionserver.HStore(327): Store=619e0c9a5ad3ede5c5b6d8d44658cb1a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:52:11,475 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:11,476 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:52:11,477 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 619e0c9a5ad3ede5c5b6d8d44658cb1a columnFamilyName C 2024-11-17T01:52:11,477 DEBUG [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:11,477 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] regionserver.HStore(327): Store=619e0c9a5ad3ede5c5b6d8d44658cb1a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:52:11,477 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:11,477 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:11,478 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:11,478 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:11,479 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:11,480 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:11,480 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T01:52:11,481 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:11,483 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:52:11,484 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 619e0c9a5ad3ede5c5b6d8d44658cb1a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72700375, jitterRate=0.0833200067281723}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T01:52:11,484 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:11,484 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: Running coprocessor pre-open hook at 1731808331466Writing region info on filesystem at 1731808331466Initializing all the Stores at 1731808331467 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808331467Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808331467Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808331468 (+1 ms)Cleaning up temporary data from old regions at 1731808331480 (+12 ms)Running coprocessor post-open hooks at 1731808331484 (+4 ms)Region opened successfully at 1731808331484 2024-11-17T01:52:11,485 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., pid=36, masterSystemTime=1731808331461 2024-11-17T01:52:11,488 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:11,488 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:11,488 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=619e0c9a5ad3ede5c5b6d8d44658cb1a, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:11,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 because future has completed 2024-11-17T01:52:11,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-17T01:52:11,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 in 184 msec 2024-11-17T01:52:11,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-17T01:52:11,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619e0c9a5ad3ede5c5b6d8d44658cb1a, ASSIGN in 342 msec 2024-11-17T01:52:11,495 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T01:52:11,495 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808331495"}]},"ts":"1731808331495"} 2024-11-17T01:52:11,497 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T01:52:11,498 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T01:52:11,500 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1880 sec 2024-11-17T01:52:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T01:52:12,475 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:12,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:12,477 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:12,477 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:12,479 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:12,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:12,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aed869b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:12,481 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366f7c77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:12,482 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:12,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:12,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:12,485 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51402, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:12,486 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T01:52:12,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is c6c0d7a0a7c0,45023,1731808291372 2024-11-17T01:52:12,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@678171c4 2024-11-17T01:52:12,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T01:52:12,489 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T01:52:12,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T01:52:12,494 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:52:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T01:52:12,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741950_1126 (size=1097) 2024-11-17T01:52:12,914 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-17T01:52:12,914 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-17T01:52:12,921 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T01:52:12,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619e0c9a5ad3ede5c5b6d8d44658cb1a, REOPEN/MOVE}] 2024-11-17T01:52:12,930 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619e0c9a5ad3ede5c5b6d8d44658cb1a, REOPEN/MOVE 2024-11-17T01:52:12,931 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=619e0c9a5ad3ede5c5b6d8d44658cb1a, regionState=CLOSING, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:12,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=619e0c9a5ad3ede5c5b6d8d44658cb1a, REOPEN/MOVE because future has completed 2024-11-17T01:52:12,933 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T01:52:12,933 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173}] 2024-11-17T01:52:13,086 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:13,086 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T01:52:13,086 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 619e0c9a5ad3ede5c5b6d8d44658cb1a, disabling compactions & flushes 2024-11-17T01:52:13,086 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:13,086 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:13,086 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. after waiting 0 ms 2024-11-17T01:52:13,086 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:13,089 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T01:52:13,090 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:13,090 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: Waiting for close lock at 1731808333086Running coprocessor pre-close hooks at 1731808333086Disabling compacts and flushes for region at 1731808333086Disabling writes for close at 1731808333086Writing region close event to WAL at 1731808333086Running coprocessor post-close hooks at 1731808333090 (+4 ms)Closed at 1731808333090 2024-11-17T01:52:13,090 WARN [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: 619e0c9a5ad3ede5c5b6d8d44658cb1a to self. 2024-11-17T01:52:13,092 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:13,092 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=619e0c9a5ad3ede5c5b6d8d44658cb1a, regionState=CLOSED 2024-11-17T01:52:13,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 because future has completed 2024-11-17T01:52:13,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-17T01:52:13,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure 619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 in 162 msec 2024-11-17T01:52:13,098 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=619e0c9a5ad3ede5c5b6d8d44658cb1a, REOPEN/MOVE; state=CLOSED, location=c6c0d7a0a7c0,41157,1731808292173; forceNewPlan=false, retain=true 2024-11-17T01:52:13,249 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=619e0c9a5ad3ede5c5b6d8d44658cb1a, regionState=OPENING, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:13,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=619e0c9a5ad3ede5c5b6d8d44658cb1a, REOPEN/MOVE because future has completed 2024-11-17T01:52:13,251 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173}] 2024-11-17T01:52:13,407 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:13,408 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:52:13,408 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:13,408 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:52:13,409 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:13,409 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:13,415 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:13,417 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:52:13,423 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 619e0c9a5ad3ede5c5b6d8d44658cb1a columnFamilyName A 2024-11-17T01:52:13,425 DEBUG [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:13,426 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] regionserver.HStore(327): Store=619e0c9a5ad3ede5c5b6d8d44658cb1a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:52:13,426 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:13,427 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:52:13,427 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 619e0c9a5ad3ede5c5b6d8d44658cb1a columnFamilyName B 2024-11-17T01:52:13,427 DEBUG [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:13,428 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] regionserver.HStore(327): Store=619e0c9a5ad3ede5c5b6d8d44658cb1a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:52:13,428 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:13,429 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:52:13,429 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 619e0c9a5ad3ede5c5b6d8d44658cb1a columnFamilyName C 2024-11-17T01:52:13,429 DEBUG [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:13,430 INFO [StoreOpener-619e0c9a5ad3ede5c5b6d8d44658cb1a-1 {}] regionserver.HStore(327): Store=619e0c9a5ad3ede5c5b6d8d44658cb1a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:52:13,430 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:13,430 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:13,431 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:13,432 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:13,433 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:13,433 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:13,434 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T01:52:13,435 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:13,436 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 619e0c9a5ad3ede5c5b6d8d44658cb1a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74968018, jitterRate=0.11711052060127258}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T01:52:13,436 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:13,438 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: Running coprocessor pre-open hook at 1731808333409Writing region info on filesystem at 1731808333409Initializing all the Stores at 1731808333410 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808333411 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808333411Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808333411Cleaning up temporary data from old regions at 1731808333433 (+22 ms)Running coprocessor post-open hooks at 1731808333436 (+3 ms)Region opened successfully at 1731808333438 (+2 ms) 2024-11-17T01:52:13,439 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., pid=41, masterSystemTime=1731808333403 2024-11-17T01:52:13,441 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:13,441 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:13,441 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=619e0c9a5ad3ede5c5b6d8d44658cb1a, regionState=OPEN, openSeqNum=5, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:13,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 because future has completed 2024-11-17T01:52:13,446 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-17T01:52:13,446 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 in 193 msec 2024-11-17T01:52:13,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-17T01:52:13,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619e0c9a5ad3ede5c5b6d8d44658cb1a, REOPEN/MOVE in 517 msec 2024-11-17T01:52:13,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-17T01:52:13,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 528 msec 2024-11-17T01:52:13,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 955 msec 2024-11-17T01:52:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-17T01:52:13,456 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:13,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,456 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:13,457 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:13,457 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:13,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:13,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:13,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aa6271d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:13,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:13,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:13,459 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:13,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a2c867e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,461 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,461 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:13,461 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:13,462 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54499fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:13,463 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:13,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f68e326, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,465 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:13,465 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:13,466 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@363a785f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:13,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:13,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:13,467 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26f9dac8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,469 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,469 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:13,469 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:13,470 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:13,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:13,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:13,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ce8229, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:13,471 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:13,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53d891f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,473 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:13,473 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:13,473 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32183627, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:13,475 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:13,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c3205fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,476 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:13,476 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:13,477 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:13,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:13,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:13,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@313478d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:13,478 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fdd044a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,480 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,480 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:13,480 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:13,481 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1882995e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:13,482 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e131fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,483 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:13,483 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:13,484 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:13,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:13,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:13,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69ef775c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:13,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:13,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:13,485 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:13,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc3611c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,486 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:13,486 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:13,487 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:13,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:13,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:13,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2834daf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:13,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:13,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:13,488 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c9429d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:13,490 DEBUG [Thread-646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:13,490 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:52:13,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-17T01:52:13,491 DEBUG [Thread-652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:13,491 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:13,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T01:52:13,492 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:13,492 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:52:13,493 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:13,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:13,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:13,493 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:13,495 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:52:13,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:52:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:13,495 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:13,495 DEBUG [Thread-643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:13,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:13,496 DEBUG [Thread-641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:13,497 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:13,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:13,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:52:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:13,505 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:13,505 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:13,506 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:13,506 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:13,506 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:13,506 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:13,506 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:13,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:52:13,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:52:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:52:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:52:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:52:13,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:52:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:52:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:52:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:13,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T01:52:13,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=A 2024-11-17T01:52:13,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:13,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=B 2024-11-17T01:52:13,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:13,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=C 2024-11-17T01:52:13,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T01:52:13,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111767a65353715542ce89e0db85ee322fd9_619e0c9a5ad3ede5c5b6d8d44658cb1a is 50, key is test_row_0/A:col10/1731808333514/Put/seqid=0 2024-11-17T01:52:13,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51480 deadline: 1731808393620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:13,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731808393622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:13,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:13,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808333616, max timestamp=1731808333627Num uniques 859; 2024-11-17T01:52:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:52:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808333616, max timestamp=1731808333627Num uniques 859; 2024-11-17T01:52:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:13,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:52:13,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808333616, max timestamp=1731808333627Num uniques 859; 2024-11-17T01:52:13,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:52:13,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T01:52:13,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:13,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:13,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:13,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:13,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:13,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741951_1127 (size=12154) 2024-11-17T01:52:13,667 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:13,673 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111767a65353715542ce89e0db85ee322fd9_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111767a65353715542ce89e0db85ee322fd9_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:13,675 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/ec69216d44034c89a85dee7fd3daf631, store: [table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:13,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808393676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:13,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:13,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/ec69216d44034c89a85dee7fd3daf631 is 175, key is test_row_0/A:col10/1731808333514/Put/seqid=0 2024-11-17T01:52:13,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808393682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:13,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808393682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:13,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741952_1128 (size=30955) 2024-11-17T01:52:13,704 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/ec69216d44034c89a85dee7fd3daf631 2024-11-17T01:52:13,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51480 deadline: 1731808393735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:13,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:13,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731808393736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:13,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/a5f1c99026bc42ad846184b15ffd22ca is 50, key is test_row_0/B:col10/1731808333514/Put/seqid=0 2024-11-17T01:52:13,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741953_1129 (size=12001) 2024-11-17T01:52:13,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/a5f1c99026bc42ad846184b15ffd22ca 2024-11-17T01:52:13,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/ef894734631c406a90fbf7555505c101 is 50, key is test_row_0/C:col10/1731808333514/Put/seqid=0 2024-11-17T01:52:13,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T01:52:13,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T01:52:13,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:13,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:13,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:13,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:13,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:13,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:13,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808393803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:13,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:13,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808393803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:13,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:13,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:13,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808393804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:13,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:13,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741954_1130 (size=12001) 2024-11-17T01:52:13,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51480 deadline: 1731808393944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:13,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731808393945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:13,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:13,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:13,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T01:52:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:13,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:13,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:14,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:14,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808394018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:14,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:14,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808394019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:14,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:14,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808394020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:14,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:14,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T01:52:14,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:14,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:14,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:14,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:14,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T01:52:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:14,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/ef894734631c406a90fbf7555505c101 2024-11-17T01:52:14,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/ec69216d44034c89a85dee7fd3daf631 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/ec69216d44034c89a85dee7fd3daf631 2024-11-17T01:52:14,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51480 deadline: 1731808394255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:14,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T01:52:14,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:14,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:14,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:14,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:14,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:14,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:14,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731808394273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:14,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:14,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/ec69216d44034c89a85dee7fd3daf631, entries=150, sequenceid=15, filesize=30.2 K 2024-11-17T01:52:14,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/a5f1c99026bc42ad846184b15ffd22ca as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a5f1c99026bc42ad846184b15ffd22ca 2024-11-17T01:52:14,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a5f1c99026bc42ad846184b15ffd22ca, entries=150, sequenceid=15, filesize=11.7 K 2024-11-17T01:52:14,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/ef894734631c406a90fbf7555505c101 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/ef894734631c406a90fbf7555505c101 2024-11-17T01:52:14,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/ef894734631c406a90fbf7555505c101, entries=150, sequenceid=15, filesize=11.7 K 2024-11-17T01:52:14,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 619e0c9a5ad3ede5c5b6d8d44658cb1a in 743ms, sequenceid=15, compaction requested=false 2024-11-17T01:52:14,313 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T01:52:14,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:14,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:14,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731808333650, max timestamp=1731808334335Num uniques 1068; 2024-11-17T01:52:14,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-17T01:52:14,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a 3/3 column families, dataSize=105.11 KB heapSize=292.59 KB 2024-11-17T01:52:14,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=A 2024-11-17T01:52:14,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:52:14,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=B 2024-11-17T01:52:14,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T01:52:14,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=C 2024-11-17T01:52:14,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T01:52:14,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111730fa010ccbd54dab8c2817d063515456_619e0c9a5ad3ede5c5b6d8d44658cb1a is 50, key is test_row_0/A:col10/1731808333650/Put/seqid=0 2024-11-17T01:52:14,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741955_1131 (size=17034) 2024-11-17T01:52:14,368 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:14,374 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111730fa010ccbd54dab8c2817d063515456_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111730fa010ccbd54dab8c2817d063515456_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:14,375 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/1392b9f817c74415944182303d9c06e0, store: [table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:14,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/1392b9f817c74415944182303d9c06e0 is 175, key is test_row_0/A:col10/1731808333650/Put/seqid=0 2024-11-17T01:52:14,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741956_1132 (size=48139) 2024-11-17T01:52:14,388 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/1392b9f817c74415944182303d9c06e0 2024-11-17T01:52:14,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/365835a01bfd4a76863db4753a179f4d is 50, key is test_row_0/B:col10/1731808333650/Put/seqid=0 2024-11-17T01:52:14,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808394401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:14,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808394404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:14,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808394409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:14,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:14,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T01:52:14,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:14,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:14,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:14,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:14,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:14,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741957_1133 (size=12001) 2024-11-17T01:52:14,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:14,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808394514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:14,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:14,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808394525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:14,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:14,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:14,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808394525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:14,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:14,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T01:52:14,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:14,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:14,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:14,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:14,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T01:52:14,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808394725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:14,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:14,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T01:52:14,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:14,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:14,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:14,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:14,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:14,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808394734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:14,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808394734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:14,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:14,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51480 deadline: 1731808394764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:14,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:14,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731808394784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:14,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/365835a01bfd4a76863db4753a179f4d 2024-11-17T01:52:14,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/dad19ea2b9df438882675d8cb018f0b6 is 50, key is test_row_0/C:col10/1731808333650/Put/seqid=0 2024-11-17T01:52:14,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741958_1134 (size=12001) 2024-11-17T01:52:14,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/dad19ea2b9df438882675d8cb018f0b6 2024-11-17T01:52:14,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/1392b9f817c74415944182303d9c06e0 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/1392b9f817c74415944182303d9c06e0 2024-11-17T01:52:14,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/1392b9f817c74415944182303d9c06e0, entries=250, sequenceid=53, filesize=47.0 K 2024-11-17T01:52:14,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/365835a01bfd4a76863db4753a179f4d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/365835a01bfd4a76863db4753a179f4d 2024-11-17T01:52:14,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T01:52:14,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:14,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:14,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:14,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:14,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:14,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/365835a01bfd4a76863db4753a179f4d, entries=150, sequenceid=53, filesize=11.7 K 2024-11-17T01:52:14,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/dad19ea2b9df438882675d8cb018f0b6 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/dad19ea2b9df438882675d8cb018f0b6 2024-11-17T01:52:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:14,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/dad19ea2b9df438882675d8cb018f0b6, entries=150, sequenceid=53, filesize=11.7 K 2024-11-17T01:52:14,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~111.82 KB/114500, heapSize ~312.47 KB/319968, currentSize=73.80 KB/75570 for 619e0c9a5ad3ede5c5b6d8d44658cb1a in 567ms, sequenceid=53, compaction requested=false 2024-11-17T01:52:14,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:15,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T01:52:15,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=A 2024-11-17T01:52:15,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:15,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=B 2024-11-17T01:52:15,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:15,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=C 2024-11-17T01:52:15,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:15,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T01:52:15,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:15,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:15,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:15,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:15,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:15,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117395281ccd19a4ade85b0c4eefd2a72fb_619e0c9a5ad3ede5c5b6d8d44658cb1a is 50, key is test_row_0/A:col10/1731808334400/Put/seqid=0 2024-11-17T01:52:15,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808395092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:15,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808395092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:15,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808395096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:15,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:15,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741959_1135 (size=14594) 2024-11-17T01:52:15,139 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:15,147 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117395281ccd19a4ade85b0c4eefd2a72fb_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117395281ccd19a4ade85b0c4eefd2a72fb_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:15,149 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/7cd0262b733d4a84b4c35bf31f4e3b58, store: [table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:15,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/7cd0262b733d4a84b4c35bf31f4e3b58 is 175, key is test_row_0/A:col10/1731808334400/Put/seqid=0 2024-11-17T01:52:15,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741960_1136 (size=39549) 2024-11-17T01:52:15,185 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/7cd0262b733d4a84b4c35bf31f4e3b58 2024-11-17T01:52:15,196 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T01:52:15,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808395205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:15,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:15,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T01:52:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:15,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:15,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/18afd26658964ea19916b5d639b4afbf is 50, key is test_row_0/B:col10/1731808334400/Put/seqid=0 2024-11-17T01:52:15,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808395214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:15,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808395223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:15,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:15,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:15,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741961_1137 (size=12001) 2024-11-17T01:52:15,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/18afd26658964ea19916b5d639b4afbf 2024-11-17T01:52:15,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/4ca0f15d92ff4c3db49a229a01d63dc0 is 50, key is test_row_0/C:col10/1731808334400/Put/seqid=0 2024-11-17T01:52:15,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741962_1138 (size=12001) 2024-11-17T01:52:15,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/4ca0f15d92ff4c3db49a229a01d63dc0 2024-11-17T01:52:15,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/7cd0262b733d4a84b4c35bf31f4e3b58 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/7cd0262b733d4a84b4c35bf31f4e3b58 2024-11-17T01:52:15,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/7cd0262b733d4a84b4c35bf31f4e3b58, entries=200, sequenceid=68, filesize=38.6 K 2024-11-17T01:52:15,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/18afd26658964ea19916b5d639b4afbf as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/18afd26658964ea19916b5d639b4afbf 2024-11-17T01:52:15,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/18afd26658964ea19916b5d639b4afbf, entries=150, sequenceid=68, filesize=11.7 K 2024-11-17T01:52:15,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/4ca0f15d92ff4c3db49a229a01d63dc0 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/4ca0f15d92ff4c3db49a229a01d63dc0 2024-11-17T01:52:15,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/4ca0f15d92ff4c3db49a229a01d63dc0, entries=150, sequenceid=68, filesize=11.7 K 2024-11-17T01:52:15,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 619e0c9a5ad3ede5c5b6d8d44658cb1a in 315ms, sequenceid=68, compaction requested=true 2024-11-17T01:52:15,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:15,355 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:15,357 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:15,357 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/A is initiating minor compaction (all files) 2024-11-17T01:52:15,357 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/A in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:15,357 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/ec69216d44034c89a85dee7fd3daf631, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/1392b9f817c74415944182303d9c06e0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/7cd0262b733d4a84b4c35bf31f4e3b58] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=115.9 K 2024-11-17T01:52:15,357 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:15,357 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/ec69216d44034c89a85dee7fd3daf631, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/1392b9f817c74415944182303d9c06e0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/7cd0262b733d4a84b4c35bf31f4e3b58] 2024-11-17T01:52:15,358 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec69216d44034c89a85dee7fd3daf631, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731808333514 2024-11-17T01:52:15,359 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1392b9f817c74415944182303d9c06e0, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731808333650 2024-11-17T01:52:15,359 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cd0262b733d4a84b4c35bf31f4e3b58, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731808334384 2024-11-17T01:52:15,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:52:15,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:15,366 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:15,367 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:15,367 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/B is initiating minor compaction (all files) 2024-11-17T01:52:15,367 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/B in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:15,367 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a5f1c99026bc42ad846184b15ffd22ca, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/365835a01bfd4a76863db4753a179f4d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/18afd26658964ea19916b5d639b4afbf] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=35.2 K 2024-11-17T01:52:15,368 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting a5f1c99026bc42ad846184b15ffd22ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731808333514 2024-11-17T01:52:15,369 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 365835a01bfd4a76863db4753a179f4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731808333650 2024-11-17T01:52:15,369 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 18afd26658964ea19916b5d639b4afbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731808334384 2024-11-17T01:52:15,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T01:52:15,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:15,377 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T01:52:15,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=A 2024-11-17T01:52:15,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:15,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=B 2024-11-17T01:52:15,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:15,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=C 2024-11-17T01:52:15,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:15,380 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:15,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:52:15,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:15,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:52:15,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:15,391 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#B#compaction#121 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:15,392 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/333b18e121cc4616b391ff8b5a0dc9e5 is 50, key is test_row_0/B:col10/1731808334400/Put/seqid=0 2024-11-17T01:52:15,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f32263ee3f7f490cba7ba9ba54615b6c_619e0c9a5ad3ede5c5b6d8d44658cb1a is 50, key is test_row_0/A:col10/1731808335081/Put/seqid=0 2024-11-17T01:52:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:15,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:15,442 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411175b945af314264bd3b5d67d8ade245dd6_619e0c9a5ad3ede5c5b6d8d44658cb1a store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:15,452 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411175b945af314264bd3b5d67d8ade245dd6_619e0c9a5ad3ede5c5b6d8d44658cb1a, store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:15,452 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175b945af314264bd3b5d67d8ade245dd6_619e0c9a5ad3ede5c5b6d8d44658cb1a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:15,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741963_1139 (size=12104) 2024-11-17T01:52:15,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808395477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:15,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808395479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:15,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:15,486 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/333b18e121cc4616b391ff8b5a0dc9e5 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/333b18e121cc4616b391ff8b5a0dc9e5 2024-11-17T01:52:15,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:15,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808395483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:15,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:15,495 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/B of 619e0c9a5ad3ede5c5b6d8d44658cb1a into 333b18e121cc4616b391ff8b5a0dc9e5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:15,495 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:15,495 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/B, priority=13, startTime=1731808335366; duration=0sec 2024-11-17T01:52:15,495 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:15,495 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:B 2024-11-17T01:52:15,495 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:15,497 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:15,497 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/C is initiating minor compaction (all files) 2024-11-17T01:52:15,497 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/C in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:15,497 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/ef894734631c406a90fbf7555505c101, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/dad19ea2b9df438882675d8cb018f0b6, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/4ca0f15d92ff4c3db49a229a01d63dc0] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=35.2 K 2024-11-17T01:52:15,498 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting ef894734631c406a90fbf7555505c101, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731808333514 2024-11-17T01:52:15,498 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting dad19ea2b9df438882675d8cb018f0b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731808333650 2024-11-17T01:52:15,499 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ca0f15d92ff4c3db49a229a01d63dc0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731808334384 2024-11-17T01:52:15,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741964_1140 (size=12154) 2024-11-17T01:52:15,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741965_1141 (size=4469) 2024-11-17T01:52:15,524 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#A#compaction#120 average throughput is 0.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:15,525 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/b08b643bec984b3e8cf1c09d04fbe3dd is 175, key is test_row_0/A:col10/1731808334400/Put/seqid=0 2024-11-17T01:52:15,529 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#C#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:15,530 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/ff45aaceb6fe4384a0b562045b199b39 is 50, key is test_row_0/C:col10/1731808334400/Put/seqid=0 2024-11-17T01:52:15,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741966_1142 (size=31058) 2024-11-17T01:52:15,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741967_1143 (size=12104) 2024-11-17T01:52:15,570 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/b08b643bec984b3e8cf1c09d04fbe3dd as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/b08b643bec984b3e8cf1c09d04fbe3dd 2024-11-17T01:52:15,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808395585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:15,590 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/A of 619e0c9a5ad3ede5c5b6d8d44658cb1a into b08b643bec984b3e8cf1c09d04fbe3dd(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:15,590 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:15,592 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/A, priority=13, startTime=1731808335355; duration=0sec 2024-11-17T01:52:15,592 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:15,592 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:A 2024-11-17T01:52:15,593 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/ff45aaceb6fe4384a0b562045b199b39 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/ff45aaceb6fe4384a0b562045b199b39 2024-11-17T01:52:15,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808395595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:15,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:15,601 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/C of 619e0c9a5ad3ede5c5b6d8d44658cb1a into ff45aaceb6fe4384a0b562045b199b39(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:15,602 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:15,602 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/C, priority=13, startTime=1731808335384; duration=0sec 2024-11-17T01:52:15,602 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:15,602 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:C 2024-11-17T01:52:15,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808395605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T01:52:15,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:15,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51480 deadline: 1731808395784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:15,788 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in 619e0c9a5ad3ede5c5b6d8d44658cb1a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:52:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:15,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808395795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:15,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:15,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808395804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:15,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:15,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:15,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731808395804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:15,812 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_2' in 619e0c9a5ad3ede5c5b6d8d44658cb1a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:52:15,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:15,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808395815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:15,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:15,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:15,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:15,909 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f32263ee3f7f490cba7ba9ba54615b6c_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f32263ee3f7f490cba7ba9ba54615b6c_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:15,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/0722c4e70eb048ae8223da516af80a63, store: [table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:15,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/0722c4e70eb048ae8223da516af80a63 is 175, key is test_row_0/A:col10/1731808335081/Put/seqid=0 2024-11-17T01:52:15,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741968_1144 (size=30955) 2024-11-17T01:52:16,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:16,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808396104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:16,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:16,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:16,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:16,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808396114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:16,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:16,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:16,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:16,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808396125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:16,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:16,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:16,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:16,325 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/0722c4e70eb048ae8223da516af80a63 2024-11-17T01:52:16,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/98cc205c84ba46dba053207a1e91e347 is 50, key is test_row_0/B:col10/1731808335081/Put/seqid=0 2024-11-17T01:52:16,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741969_1145 (size=12001) 2024-11-17T01:52:16,399 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/98cc205c84ba46dba053207a1e91e347 2024-11-17T01:52:16,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/73ecf5ab05494337ac3e28e689b7999a is 50, key is test_row_0/C:col10/1731808335081/Put/seqid=0 2024-11-17T01:52:16,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741970_1146 (size=12001) 2024-11-17T01:52:16,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:16,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808396614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:16,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808396625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:16,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808396634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:16,731 INFO [master/c6c0d7a0a7c0:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-17T01:52:16,731 INFO [master/c6c0d7a0a7c0:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-17T01:52:16,815 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/73ecf5ab05494337ac3e28e689b7999a 2024-11-17T01:52:16,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/0722c4e70eb048ae8223da516af80a63 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/0722c4e70eb048ae8223da516af80a63 2024-11-17T01:52:16,834 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/0722c4e70eb048ae8223da516af80a63, entries=150, sequenceid=85, filesize=30.2 K 2024-11-17T01:52:16,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/98cc205c84ba46dba053207a1e91e347 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/98cc205c84ba46dba053207a1e91e347 2024-11-17T01:52:16,841 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/98cc205c84ba46dba053207a1e91e347, entries=150, sequenceid=85, filesize=11.7 K 2024-11-17T01:52:16,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/73ecf5ab05494337ac3e28e689b7999a as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/73ecf5ab05494337ac3e28e689b7999a 2024-11-17T01:52:16,848 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/73ecf5ab05494337ac3e28e689b7999a, entries=150, sequenceid=85, filesize=11.7 K 2024-11-17T01:52:16,849 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 619e0c9a5ad3ede5c5b6d8d44658cb1a in 1472ms, sequenceid=85, compaction requested=false 2024-11-17T01:52:16,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:16,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:16,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-17T01:52:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-17T01:52:16,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-17T01:52:16,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3560 sec 2024-11-17T01:52:16,856 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 3.3640 sec 2024-11-17T01:52:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:17,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T01:52:17,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=A 2024-11-17T01:52:17,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:17,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=B 2024-11-17T01:52:17,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:17,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=C 2024-11-17T01:52:17,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:17,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175aa8a4a6e90b4949bf318ab029e4da76_619e0c9a5ad3ede5c5b6d8d44658cb1a is 50, key is test_row_0/A:col10/1731808335476/Put/seqid=0 2024-11-17T01:52:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T01:52:17,655 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:17,656 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:52:17,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-17T01:52:17,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T01:52:17,661 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:52:17,662 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:52:17,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:52:17,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:17,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808397686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:17,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:17,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808397692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:17,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:17,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808397692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:17,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741971_1147 (size=12154) 2024-11-17T01:52:17,701 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:17,708 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175aa8a4a6e90b4949bf318ab029e4da76_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175aa8a4a6e90b4949bf318ab029e4da76_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:17,711 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/f22a4e6372574d10bd154bc4fe2b26a9, store: [table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:17,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/f22a4e6372574d10bd154bc4fe2b26a9 is 175, key is test_row_0/A:col10/1731808335476/Put/seqid=0 2024-11-17T01:52:17,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741972_1148 (size=30955) 2024-11-17T01:52:17,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T01:52:17,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51480 deadline: 1731808397805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:17,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808397805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:17,807 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in 619e0c9a5ad3ede5c5b6d8d44658cb1a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:52:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:17,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808397806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:17,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808397806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:17,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:17,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:17,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:17,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:17,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:17,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:17,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T01:52:17,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:17,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:17,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:17,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:17,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:17,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731808397824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:17,827 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_2' in 619e0c9a5ad3ede5c5b6d8d44658cb1a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:52:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:17,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T01:52:17,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:17,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:17,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:17,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:17,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T01:52:18,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808398014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:18,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808398015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808398016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:18,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T01:52:18,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:18,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:18,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:18,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:18,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:18,143 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/f22a4e6372574d10bd154bc4fe2b26a9 2024-11-17T01:52:18,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/b2891bd54f454cd1ae10872f2bcd6439 is 50, key is test_row_0/B:col10/1731808335476/Put/seqid=0 2024-11-17T01:52:18,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741973_1149 (size=12001) 2024-11-17T01:52:18,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/b2891bd54f454cd1ae10872f2bcd6439 2024-11-17T01:52:18,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/76f2a413fe9b468db1ebee83d1aa6321 is 50, key is test_row_0/C:col10/1731808335476/Put/seqid=0 2024-11-17T01:52:18,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741974_1150 (size=12001) 2024-11-17T01:52:18,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T01:52:18,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:18,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:18,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:18,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:18,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T01:52:18,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808398324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:18,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808398324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:18,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808398324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:18,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T01:52:18,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:18,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:18,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:18,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:18,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:18,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T01:52:18,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:18,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:18,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:18,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:18,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:18,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/76f2a413fe9b468db1ebee83d1aa6321 2024-11-17T01:52:18,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/f22a4e6372574d10bd154bc4fe2b26a9 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/f22a4e6372574d10bd154bc4fe2b26a9 2024-11-17T01:52:18,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/f22a4e6372574d10bd154bc4fe2b26a9, entries=150, sequenceid=104, filesize=30.2 K 2024-11-17T01:52:18,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/b2891bd54f454cd1ae10872f2bcd6439 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/b2891bd54f454cd1ae10872f2bcd6439 2024-11-17T01:52:18,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/b2891bd54f454cd1ae10872f2bcd6439, entries=150, sequenceid=104, filesize=11.7 K 2024-11-17T01:52:18,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/76f2a413fe9b468db1ebee83d1aa6321 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/76f2a413fe9b468db1ebee83d1aa6321 2024-11-17T01:52:18,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/76f2a413fe9b468db1ebee83d1aa6321, entries=150, sequenceid=104, filesize=11.7 K 2024-11-17T01:52:18,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 619e0c9a5ad3ede5c5b6d8d44658cb1a in 983ms, sequenceid=104, compaction requested=true 2024-11-17T01:52:18,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:18,624 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:18,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:52:18,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:18,625 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:18,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:52:18,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:18,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:52:18,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:18,628 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:18,628 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:18,628 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/A is initiating minor compaction (all files) 2024-11-17T01:52:18,628 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/B is initiating minor compaction (all files) 2024-11-17T01:52:18,628 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/B in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:18,628 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/333b18e121cc4616b391ff8b5a0dc9e5, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/98cc205c84ba46dba053207a1e91e347, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/b2891bd54f454cd1ae10872f2bcd6439] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=35.3 K 2024-11-17T01:52:18,629 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/A in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:18,629 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/b08b643bec984b3e8cf1c09d04fbe3dd, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/0722c4e70eb048ae8223da516af80a63, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/f22a4e6372574d10bd154bc4fe2b26a9] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=90.8 K 2024-11-17T01:52:18,629 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:18,629 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/b08b643bec984b3e8cf1c09d04fbe3dd, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/0722c4e70eb048ae8223da516af80a63, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/f22a4e6372574d10bd154bc4fe2b26a9] 2024-11-17T01:52:18,629 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 333b18e121cc4616b391ff8b5a0dc9e5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731808334384 2024-11-17T01:52:18,630 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 98cc205c84ba46dba053207a1e91e347, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731808335081 2024-11-17T01:52:18,630 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting b08b643bec984b3e8cf1c09d04fbe3dd, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731808334384 2024-11-17T01:52:18,630 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting b2891bd54f454cd1ae10872f2bcd6439, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731808335474 2024-11-17T01:52:18,631 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0722c4e70eb048ae8223da516af80a63, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731808335081 2024-11-17T01:52:18,632 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting f22a4e6372574d10bd154bc4fe2b26a9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731808335474 2024-11-17T01:52:18,649 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:18,653 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#B#compaction#130 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:18,654 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/a8162ca9f0394d69af087e0f475ba54f is 50, key is test_row_0/B:col10/1731808335476/Put/seqid=0 2024-11-17T01:52:18,654 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411176cfdd3b905f24c83ae157594dae93173_619e0c9a5ad3ede5c5b6d8d44658cb1a store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:18,658 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411176cfdd3b905f24c83ae157594dae93173_619e0c9a5ad3ede5c5b6d8d44658cb1a, store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:18,658 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176cfdd3b905f24c83ae157594dae93173_619e0c9a5ad3ede5c5b6d8d44658cb1a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:18,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741975_1151 (size=4469) 2024-11-17T01:52:18,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741976_1152 (size=12207) 2024-11-17T01:52:18,694 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/a8162ca9f0394d69af087e0f475ba54f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a8162ca9f0394d69af087e0f475ba54f 2024-11-17T01:52:18,703 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/B of 619e0c9a5ad3ede5c5b6d8d44658cb1a into a8162ca9f0394d69af087e0f475ba54f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:18,703 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:18,703 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/B, priority=13, startTime=1731808338625; duration=0sec 2024-11-17T01:52:18,703 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:18,703 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:B 2024-11-17T01:52:18,704 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:18,705 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:18,705 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/C is initiating minor compaction (all files) 2024-11-17T01:52:18,705 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/C in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:18,705 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/ff45aaceb6fe4384a0b562045b199b39, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/73ecf5ab05494337ac3e28e689b7999a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/76f2a413fe9b468db1ebee83d1aa6321] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=35.3 K 2024-11-17T01:52:18,705 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting ff45aaceb6fe4384a0b562045b199b39, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731808334384 2024-11-17T01:52:18,707 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 73ecf5ab05494337ac3e28e689b7999a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731808335081 2024-11-17T01:52:18,709 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 76f2a413fe9b468db1ebee83d1aa6321, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731808335474 2024-11-17T01:52:18,721 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#C#compaction#131 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:18,722 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/daa3622ac2b34adcba975422a05c8b19 is 50, key is test_row_0/C:col10/1731808335476/Put/seqid=0 2024-11-17T01:52:18,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741977_1153 (size=12207) 2024-11-17T01:52:18,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T01:52:18,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:18,740 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/daa3622ac2b34adcba975422a05c8b19 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/daa3622ac2b34adcba975422a05c8b19 2024-11-17T01:52:18,740 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T01:52:18,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=A 2024-11-17T01:52:18,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:18,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=B 2024-11-17T01:52:18,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:18,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=C 2024-11-17T01:52:18,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:18,749 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/C of 619e0c9a5ad3ede5c5b6d8d44658cb1a into daa3622ac2b34adcba975422a05c8b19(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:18,749 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:18,749 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/C, priority=13, startTime=1731808338625; duration=0sec 2024-11-17T01:52:18,749 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:18,750 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:C 2024-11-17T01:52:18,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a706e35a5339481fabfd0b1b4d3d6c68_619e0c9a5ad3ede5c5b6d8d44658cb1a is 50, key is test_row_0/A:col10/1731808337689/Put/seqid=0 2024-11-17T01:52:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T01:52:18,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741978_1154 (size=12154) 2024-11-17T01:52:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:18,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:18,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808398859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:18,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:18,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808398860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:18,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:18,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:18,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808398862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:18,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:18,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808398964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:18,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:18,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808398974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:18,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808398974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:19,074 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#A#compaction#129 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:19,075 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/26e4ca2dd1b5442eb50eb608f8c1315e is 175, key is test_row_0/A:col10/1731808335476/Put/seqid=0 2024-11-17T01:52:19,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741979_1155 (size=31161) 2024-11-17T01:52:19,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808399174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:19,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:19,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:19,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:19,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:19,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808399184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:19,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:19,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:19,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:19,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808399184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:19,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:19,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:19,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:19,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:19,215 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a706e35a5339481fabfd0b1b4d3d6c68_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a706e35a5339481fabfd0b1b4d3d6c68_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:19,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/f67b15549d184fa5ae9503be14c41eb2, store: [table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:19,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/f67b15549d184fa5ae9503be14c41eb2 is 175, key is test_row_0/A:col10/1731808337689/Put/seqid=0 2024-11-17T01:52:19,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741980_1156 (size=30955) 2024-11-17T01:52:19,221 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/f67b15549d184fa5ae9503be14c41eb2 2024-11-17T01:52:19,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/fbf1032a139746cbaf53184c7b672cd2 is 50, key is test_row_0/B:col10/1731808337689/Put/seqid=0 2024-11-17T01:52:19,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741981_1157 (size=12001) 2024-11-17T01:52:19,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:19,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808399484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:19,491 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/26e4ca2dd1b5442eb50eb608f8c1315e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/26e4ca2dd1b5442eb50eb608f8c1315e 2024-11-17T01:52:19,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808399495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:19,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:19,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808399495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:19,496 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/A of 619e0c9a5ad3ede5c5b6d8d44658cb1a into 26e4ca2dd1b5442eb50eb608f8c1315e(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:19,496 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:19,496 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/A, priority=13, startTime=1731808338624; duration=0sec 2024-11-17T01:52:19,497 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:19,498 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:A 2024-11-17T01:52:19,639 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/fbf1032a139746cbaf53184c7b672cd2 2024-11-17T01:52:19,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/ce1396d0aff7419e8db93c4d6b9309fc is 50, key is test_row_0/C:col10/1731808337689/Put/seqid=0 2024-11-17T01:52:19,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741982_1158 (size=12001) 2024-11-17T01:52:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T01:52:19,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:19,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808399995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:19,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:19,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:19,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:20,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808400004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:20,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808400004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:20,058 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/ce1396d0aff7419e8db93c4d6b9309fc 2024-11-17T01:52:20,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/f67b15549d184fa5ae9503be14c41eb2 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/f67b15549d184fa5ae9503be14c41eb2 2024-11-17T01:52:20,068 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/f67b15549d184fa5ae9503be14c41eb2, entries=150, sequenceid=121, filesize=30.2 K 2024-11-17T01:52:20,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/fbf1032a139746cbaf53184c7b672cd2 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/fbf1032a139746cbaf53184c7b672cd2 2024-11-17T01:52:20,074 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/fbf1032a139746cbaf53184c7b672cd2, entries=150, sequenceid=121, filesize=11.7 K 2024-11-17T01:52:20,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/ce1396d0aff7419e8db93c4d6b9309fc as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/ce1396d0aff7419e8db93c4d6b9309fc 2024-11-17T01:52:20,081 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/ce1396d0aff7419e8db93c4d6b9309fc, entries=150, sequenceid=121, filesize=11.7 K 2024-11-17T01:52:20,082 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 619e0c9a5ad3ede5c5b6d8d44658cb1a in 1342ms, sequenceid=121, compaction requested=false 2024-11-17T01:52:20,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:20,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:20,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-17T01:52:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-17T01:52:20,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-17T01:52:20,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4220 sec 2024-11-17T01:52:20,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.4310 sec 2024-11-17T01:52:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:21,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T01:52:21,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=A 2024-11-17T01:52:21,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:21,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=B 2024-11-17T01:52:21,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:21,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=C 2024-11-17T01:52:21,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:21,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117356fc0d3adc2472d9ea9a0ad7e2b60ee_619e0c9a5ad3ede5c5b6d8d44658cb1a is 50, key is test_row_0/A:col10/1731808341006/Put/seqid=0 2024-11-17T01:52:21,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741983_1159 (size=14744) 2024-11-17T01:52:21,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808401031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:21,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808401035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:21,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808401036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:21,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808401144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:21,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808401144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:21,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:21,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:21,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808401144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:21,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:21,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:21,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808401354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:21,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808401354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:21,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808401354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:21,420 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:21,424 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117356fc0d3adc2472d9ea9a0ad7e2b60ee_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117356fc0d3adc2472d9ea9a0ad7e2b60ee_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:21,425 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/b43903a6a40c4fa18ab46248f131b450, store: [table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:21,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/b43903a6a40c4fa18ab46248f131b450 is 175, key is test_row_0/A:col10/1731808341006/Put/seqid=0 2024-11-17T01:52:21,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741984_1160 (size=39699) 2024-11-17T01:52:21,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808401664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:21,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:21,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808401664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:21,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:21,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808401665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T01:52:21,824 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:21,825 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:52:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-17T01:52:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T01:52:21,826 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:52:21,827 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:52:21,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:52:21,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51480 deadline: 1731808401824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:21,829 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in 619e0c9a5ad3ede5c5b6d8d44658cb1a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:52:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:21,831 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/b43903a6a40c4fa18ab46248f131b450 2024-11-17T01:52:21,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/a89e41895f4240bbbed4cab978cd7ad7 is 50, key is test_row_0/B:col10/1731808341006/Put/seqid=0 2024-11-17T01:52:21,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741985_1161 (size=12101) 2024-11-17T01:52:21,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:21,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731808401865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:21,866 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_2' in 619e0c9a5ad3ede5c5b6d8d44658cb1a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:52:21,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:21,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T01:52:21,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T01:52:21,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:21,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:21,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:21,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:21,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:22,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T01:52:22,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:22,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:22,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:22,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:22,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T01:52:22,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808402174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:22,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:22,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:22,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:22,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:22,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808402184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808402184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:22,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/a89e41895f4240bbbed4cab978cd7ad7 2024-11-17T01:52:22,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/e5446c498e8c41e99f94cc48e50103ba is 50, key is test_row_0/C:col10/1731808341006/Put/seqid=0 2024-11-17T01:52:22,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741986_1162 (size=12101) 2024-11-17T01:52:22,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T01:52:22,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:22,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:22,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:22,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:22,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:22,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T01:52:22,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:22,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:22,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:22,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:22,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T01:52:22,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T01:52:22,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:22,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:22,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:22,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:22,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:22,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/e5446c498e8c41e99f94cc48e50103ba 2024-11-17T01:52:22,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/b43903a6a40c4fa18ab46248f131b450 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/b43903a6a40c4fa18ab46248f131b450 2024-11-17T01:52:22,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/b43903a6a40c4fa18ab46248f131b450, entries=200, sequenceid=140, filesize=38.8 K 2024-11-17T01:52:22,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/a89e41895f4240bbbed4cab978cd7ad7 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a89e41895f4240bbbed4cab978cd7ad7 2024-11-17T01:52:22,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a89e41895f4240bbbed4cab978cd7ad7, entries=150, sequenceid=140, filesize=11.8 K 2024-11-17T01:52:22,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/e5446c498e8c41e99f94cc48e50103ba as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/e5446c498e8c41e99f94cc48e50103ba 2024-11-17T01:52:22,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/e5446c498e8c41e99f94cc48e50103ba, entries=150, sequenceid=140, filesize=11.8 K 2024-11-17T01:52:22,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 619e0c9a5ad3ede5c5b6d8d44658cb1a in 1688ms, sequenceid=140, compaction requested=true 2024-11-17T01:52:22,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:22,697 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:22,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:52:22,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:22,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:52:22,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:22,697 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:22,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:52:22,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:22,701 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:22,701 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:22,701 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/A is initiating minor compaction (all files) 2024-11-17T01:52:22,701 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/B is initiating minor compaction (all files) 2024-11-17T01:52:22,701 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/B in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:22,701 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/A in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:22,701 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/26e4ca2dd1b5442eb50eb608f8c1315e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/f67b15549d184fa5ae9503be14c41eb2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/b43903a6a40c4fa18ab46248f131b450] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=99.4 K 2024-11-17T01:52:22,701 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a8162ca9f0394d69af087e0f475ba54f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/fbf1032a139746cbaf53184c7b672cd2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a89e41895f4240bbbed4cab978cd7ad7] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=35.5 K 2024-11-17T01:52:22,701 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:22,701 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/26e4ca2dd1b5442eb50eb608f8c1315e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/f67b15549d184fa5ae9503be14c41eb2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/b43903a6a40c4fa18ab46248f131b450] 2024-11-17T01:52:22,702 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting a8162ca9f0394d69af087e0f475ba54f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731808335474 2024-11-17T01:52:22,702 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26e4ca2dd1b5442eb50eb608f8c1315e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731808335474 2024-11-17T01:52:22,702 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting fbf1032a139746cbaf53184c7b672cd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731808337680 2024-11-17T01:52:22,702 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting f67b15549d184fa5ae9503be14c41eb2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731808337680 2024-11-17T01:52:22,703 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting b43903a6a40c4fa18ab46248f131b450, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731808338837 2024-11-17T01:52:22,703 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting a89e41895f4240bbbed4cab978cd7ad7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731808338837 2024-11-17T01:52:22,712 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:22,714 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117e33c92c67cc745569d2cd4cdc49a5dd5_619e0c9a5ad3ede5c5b6d8d44658cb1a store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:22,715 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#B#compaction#139 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:22,716 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/bc16cefd15094dcf9511a5e67954c23f is 50, key is test_row_0/B:col10/1731808341006/Put/seqid=0 2024-11-17T01:52:22,717 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117e33c92c67cc745569d2cd4cdc49a5dd5_619e0c9a5ad3ede5c5b6d8d44658cb1a, store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:22,717 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e33c92c67cc745569d2cd4cdc49a5dd5_619e0c9a5ad3ede5c5b6d8d44658cb1a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:22,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741988_1164 (size=12409) 2024-11-17T01:52:22,736 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/bc16cefd15094dcf9511a5e67954c23f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/bc16cefd15094dcf9511a5e67954c23f 2024-11-17T01:52:22,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T01:52:22,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:22,746 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T01:52:22,747 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/B of 619e0c9a5ad3ede5c5b6d8d44658cb1a into bc16cefd15094dcf9511a5e67954c23f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:22,747 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:22,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=A 2024-11-17T01:52:22,747 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/B, priority=13, startTime=1731808342697; duration=0sec 2024-11-17T01:52:22,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:22,747 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:22,747 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:B 2024-11-17T01:52:22,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=B 2024-11-17T01:52:22,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:22,747 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:22,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=C 2024-11-17T01:52:22,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:22,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741987_1163 (size=4469) 2024-11-17T01:52:22,749 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:22,749 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/C is initiating minor compaction (all files) 2024-11-17T01:52:22,749 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/C in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:22,749 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/daa3622ac2b34adcba975422a05c8b19, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/ce1396d0aff7419e8db93c4d6b9309fc, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/e5446c498e8c41e99f94cc48e50103ba] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=35.5 K 2024-11-17T01:52:22,751 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting daa3622ac2b34adcba975422a05c8b19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731808335474 2024-11-17T01:52:22,752 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting ce1396d0aff7419e8db93c4d6b9309fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731808337680 2024-11-17T01:52:22,755 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting e5446c498e8c41e99f94cc48e50103ba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731808338837 2024-11-17T01:52:22,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111705ac2e2fd7a14687af80b1d1e78d7f24_619e0c9a5ad3ede5c5b6d8d44658cb1a is 50, key is test_row_0/A:col10/1731808341034/Put/seqid=0 2024-11-17T01:52:22,770 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:22,770 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/499e06c1f40741599d528a46f34d1d0a is 50, key is test_row_0/C:col10/1731808341006/Put/seqid=0 2024-11-17T01:52:22,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741989_1165 (size=12304) 2024-11-17T01:52:22,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:22,784 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111705ac2e2fd7a14687af80b1d1e78d7f24_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111705ac2e2fd7a14687af80b1d1e78d7f24_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:22,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/fa239086aab547b2ad48d8f9ab9936c4, store: [table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:22,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/fa239086aab547b2ad48d8f9ab9936c4 is 175, key is test_row_0/A:col10/1731808341034/Put/seqid=0 2024-11-17T01:52:22,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741990_1166 (size=12409) 2024-11-17T01:52:22,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741991_1167 (size=31105) 2024-11-17T01:52:22,797 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/499e06c1f40741599d528a46f34d1d0a as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/499e06c1f40741599d528a46f34d1d0a 2024-11-17T01:52:22,797 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/fa239086aab547b2ad48d8f9ab9936c4 2024-11-17T01:52:22,803 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/C of 619e0c9a5ad3ede5c5b6d8d44658cb1a into 499e06c1f40741599d528a46f34d1d0a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:22,803 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:22,804 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/C, priority=13, startTime=1731808342697; duration=0sec 2024-11-17T01:52:22,804 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:22,804 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:C 2024-11-17T01:52:22,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/6522972687fb47eab5e36e5c3a0a7334 is 50, key is test_row_0/B:col10/1731808341034/Put/seqid=0 2024-11-17T01:52:22,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741992_1168 (size=12151) 2024-11-17T01:52:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T01:52:23,149 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#A#compaction#138 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:23,149 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/6ae8c476c110496a888b2cba9d60b0e2 is 175, key is test_row_0/A:col10/1731808341006/Put/seqid=0 2024-11-17T01:52:23,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741993_1169 (size=31363) 2024-11-17T01:52:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:23,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:23,214 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/6522972687fb47eab5e36e5c3a0a7334 2024-11-17T01:52:23,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/c91beacd3c7248b784767e926cccb187 is 50, key is test_row_0/C:col10/1731808341034/Put/seqid=0 2024-11-17T01:52:23,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808403217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:23,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808403218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:23,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:23,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:23,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808403223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:23,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741994_1170 (size=12151) 2024-11-17T01:52:23,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808403334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:23,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808403334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808403334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:23,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808403544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:23,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:23,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808403544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:23,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:23,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808403544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:23,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:23,561 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/6ae8c476c110496a888b2cba9d60b0e2 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/6ae8c476c110496a888b2cba9d60b0e2 2024-11-17T01:52:23,566 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/A of 619e0c9a5ad3ede5c5b6d8d44658cb1a into 6ae8c476c110496a888b2cba9d60b0e2(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:23,566 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:23,566 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/A, priority=13, startTime=1731808342697; duration=0sec 2024-11-17T01:52:23,566 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:23,566 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:A 2024-11-17T01:52:23,638 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/c91beacd3c7248b784767e926cccb187 2024-11-17T01:52:23,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/fa239086aab547b2ad48d8f9ab9936c4 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/fa239086aab547b2ad48d8f9ab9936c4 2024-11-17T01:52:23,648 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/fa239086aab547b2ad48d8f9ab9936c4, entries=150, sequenceid=156, filesize=30.4 K 2024-11-17T01:52:23,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/6522972687fb47eab5e36e5c3a0a7334 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/6522972687fb47eab5e36e5c3a0a7334 2024-11-17T01:52:23,654 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/6522972687fb47eab5e36e5c3a0a7334, entries=150, sequenceid=156, filesize=11.9 K 2024-11-17T01:52:23,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/c91beacd3c7248b784767e926cccb187 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/c91beacd3c7248b784767e926cccb187 2024-11-17T01:52:23,663 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/c91beacd3c7248b784767e926cccb187, entries=150, sequenceid=156, filesize=11.9 K 2024-11-17T01:52:23,664 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 619e0c9a5ad3ede5c5b6d8d44658cb1a in 918ms, sequenceid=156, compaction requested=false 2024-11-17T01:52:23,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:23,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:23,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-17T01:52:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-17T01:52:23,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-17T01:52:23,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8380 sec 2024-11-17T01:52:23,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.8430 sec 2024-11-17T01:52:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:23,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T01:52:23,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=A 2024-11-17T01:52:23,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:23,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=B 2024-11-17T01:52:23,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:23,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=C 2024-11-17T01:52:23,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:23,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808403874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:23,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808403876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:23,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:23,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178099c2b9147f476eb4fad723de67db86_619e0c9a5ad3ede5c5b6d8d44658cb1a is 50, key is test_row_0/A:col10/1731808343859/Put/seqid=0 2024-11-17T01:52:23,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:23,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808403880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:23,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:23,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741995_1171 (size=12304) 2024-11-17T01:52:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T01:52:23,975 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:23,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:52:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-17T01:52:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T01:52:23,977 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:52:23,978 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:52:23,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:52:23,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808403984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:23,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808403994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:23,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808403994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T01:52:24,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T01:52:24,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:24,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:24,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:24,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:24,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:24,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808404195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:24,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:24,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:24,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:24,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808404204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:24,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:24,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:24,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808404204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:24,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:24,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T01:52:24,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:24,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:24,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:24,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:24,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:24,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T01:52:24,308 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:24,315 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178099c2b9147f476eb4fad723de67db86_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178099c2b9147f476eb4fad723de67db86_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:24,317 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/6a4c9520593342c5b19d79226c9fd6f8, store: [table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:24,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/6a4c9520593342c5b19d79226c9fd6f8 is 175, key is test_row_0/A:col10/1731808343859/Put/seqid=0 2024-11-17T01:52:24,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741996_1172 (size=31105) 2024-11-17T01:52:24,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T01:52:24,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:24,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:24,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:24,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:24,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:24,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808404504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:24,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808404514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:24,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808404515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:24,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:24,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T01:52:24,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:24,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:24,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:24,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:24,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T01:52:24,722 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=178, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/6a4c9520593342c5b19d79226c9fd6f8 2024-11-17T01:52:24,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/0bbb3e5d07df482198297e799dff0ebf is 50, key is test_row_0/B:col10/1731808343859/Put/seqid=0 2024-11-17T01:52:24,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741997_1173 (size=12151) 2024-11-17T01:52:24,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T01:52:24,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:24,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:24,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:24,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:24,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:24,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T01:52:24,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:24,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:24,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:24,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:24,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:25,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808405023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:25,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808405024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:25,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808405024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:25,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T01:52:25,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:25,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:25,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:25,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:25,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T01:52:25,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/0bbb3e5d07df482198297e799dff0ebf 2024-11-17T01:52:25,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/298fffbf7de647038428438953c01f20 is 50, key is test_row_0/C:col10/1731808343859/Put/seqid=0 2024-11-17T01:52:25,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741998_1174 (size=12151) 2024-11-17T01:52:25,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T01:52:25,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:25,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:25,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:25,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:25,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:25,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:25,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T01:52:25,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:25,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:25,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:25,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:25,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:25,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T01:52:25,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:25,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:25,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:25,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:25,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:25,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/298fffbf7de647038428438953c01f20 2024-11-17T01:52:25,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/6a4c9520593342c5b19d79226c9fd6f8 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/6a4c9520593342c5b19d79226c9fd6f8 2024-11-17T01:52:25,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/6a4c9520593342c5b19d79226c9fd6f8, entries=150, sequenceid=178, filesize=30.4 K 2024-11-17T01:52:25,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/0bbb3e5d07df482198297e799dff0ebf as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/0bbb3e5d07df482198297e799dff0ebf 2024-11-17T01:52:25,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/0bbb3e5d07df482198297e799dff0ebf, entries=150, sequenceid=178, filesize=11.9 K 2024-11-17T01:52:25,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/298fffbf7de647038428438953c01f20 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/298fffbf7de647038428438953c01f20 2024-11-17T01:52:25,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/298fffbf7de647038428438953c01f20, entries=150, sequenceid=178, filesize=11.9 K 2024-11-17T01:52:25,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 619e0c9a5ad3ede5c5b6d8d44658cb1a in 1720ms, sequenceid=178, compaction requested=true 2024-11-17T01:52:25,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:25,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:52:25,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:25,579 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:25,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:52:25,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:25,579 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:25,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:52:25,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:25,581 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93573 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:25,581 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:25,581 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/B is initiating minor compaction (all files) 2024-11-17T01:52:25,581 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/A is initiating minor compaction (all files) 2024-11-17T01:52:25,581 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/A in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:25,581 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/B in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:25,581 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/6ae8c476c110496a888b2cba9d60b0e2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/fa239086aab547b2ad48d8f9ab9936c4, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/6a4c9520593342c5b19d79226c9fd6f8] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=91.4 K 2024-11-17T01:52:25,581 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/bc16cefd15094dcf9511a5e67954c23f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/6522972687fb47eab5e36e5c3a0a7334, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/0bbb3e5d07df482198297e799dff0ebf] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=35.9 K 2024-11-17T01:52:25,581 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:25,581 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/6ae8c476c110496a888b2cba9d60b0e2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/fa239086aab547b2ad48d8f9ab9936c4, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/6a4c9520593342c5b19d79226c9fd6f8] 2024-11-17T01:52:25,582 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting bc16cefd15094dcf9511a5e67954c23f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731808338837 2024-11-17T01:52:25,582 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ae8c476c110496a888b2cba9d60b0e2, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731808338837 2024-11-17T01:52:25,582 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 6522972687fb47eab5e36e5c3a0a7334, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731808341030 2024-11-17T01:52:25,582 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa239086aab547b2ad48d8f9ab9936c4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731808341030 2024-11-17T01:52:25,582 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bbb3e5d07df482198297e799dff0ebf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731808343222 2024-11-17T01:52:25,583 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a4c9520593342c5b19d79226c9fd6f8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731808343222 2024-11-17T01:52:25,589 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:25,592 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#B#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:25,592 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/3767cfc658c94b49a49d499cbb512ad3 is 50, key is test_row_0/B:col10/1731808343859/Put/seqid=0 2024-11-17T01:52:25,594 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411170a8682b36f114a269ceb8374394ac0e7_619e0c9a5ad3ede5c5b6d8d44658cb1a store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:25,596 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411170a8682b36f114a269ceb8374394ac0e7_619e0c9a5ad3ede5c5b6d8d44658cb1a, store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:25,596 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170a8682b36f114a269ceb8374394ac0e7_619e0c9a5ad3ede5c5b6d8d44658cb1a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:25,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741999_1175 (size=12561) 2024-11-17T01:52:25,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742000_1176 (size=4469) 2024-11-17T01:52:25,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T01:52:25,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:25,660 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T01:52:25,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=A 2024-11-17T01:52:25,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:25,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=B 2024-11-17T01:52:25,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:25,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=C 2024-11-17T01:52:25,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:25,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b3ab4063ba1742ffa787f6dd0784252b_619e0c9a5ad3ede5c5b6d8d44658cb1a is 50, key is test_row_0/A:col10/1731808343873/Put/seqid=0 2024-11-17T01:52:25,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742001_1177 (size=12304) 2024-11-17T01:52:25,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:25,680 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b3ab4063ba1742ffa787f6dd0784252b_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b3ab4063ba1742ffa787f6dd0784252b_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:25,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/a1705ce3a47342228f821ffffa0e11e1, store: [table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:25,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/a1705ce3a47342228f821ffffa0e11e1 is 175, key is test_row_0/A:col10/1731808343873/Put/seqid=0 2024-11-17T01:52:25,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742002_1178 (size=31105) 2024-11-17T01:52:25,693 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/a1705ce3a47342228f821ffffa0e11e1 2024-11-17T01:52:25,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/9f33597a6a5f486cb4e4c5a935126c9e is 50, key is test_row_0/B:col10/1731808343873/Put/seqid=0 2024-11-17T01:52:25,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742003_1179 (size=12151) 2024-11-17T01:52:26,005 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/3767cfc658c94b49a49d499cbb512ad3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/3767cfc658c94b49a49d499cbb512ad3 2024-11-17T01:52:26,007 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#A#compaction#148 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:26,008 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/4393d9ef630d402a8b475a2212912ce3 is 175, key is test_row_0/A:col10/1731808343859/Put/seqid=0 2024-11-17T01:52:26,012 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/B of 619e0c9a5ad3ede5c5b6d8d44658cb1a into 3767cfc658c94b49a49d499cbb512ad3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:26,012 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:26,012 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/B, priority=13, startTime=1731808345579; duration=0sec 2024-11-17T01:52:26,012 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:26,012 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:B 2024-11-17T01:52:26,012 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:26,016 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:26,016 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/C is initiating minor compaction (all files) 2024-11-17T01:52:26,016 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/C in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:26,016 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/499e06c1f40741599d528a46f34d1d0a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/c91beacd3c7248b784767e926cccb187, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/298fffbf7de647038428438953c01f20] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=35.9 K 2024-11-17T01:52:26,017 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 499e06c1f40741599d528a46f34d1d0a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731808338837 2024-11-17T01:52:26,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742004_1180 (size=31515) 2024-11-17T01:52:26,017 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting c91beacd3c7248b784767e926cccb187, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731808341030 2024-11-17T01:52:26,018 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 298fffbf7de647038428438953c01f20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731808343222 2024-11-17T01:52:26,027 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/4393d9ef630d402a8b475a2212912ce3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/4393d9ef630d402a8b475a2212912ce3 2024-11-17T01:52:26,037 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/A of 619e0c9a5ad3ede5c5b6d8d44658cb1a into 4393d9ef630d402a8b475a2212912ce3(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:26,037 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:26,037 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/A, priority=13, startTime=1731808345579; duration=0sec 2024-11-17T01:52:26,037 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:26,037 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:A 2024-11-17T01:52:26,038 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#C#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:26,038 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/41cd3899f765425b92a8c73d2ed5d4c7 is 50, key is test_row_0/C:col10/1731808343859/Put/seqid=0 2024-11-17T01:52:26,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742005_1181 (size=12561) 2024-11-17T01:52:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:26,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:26,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808406066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:26,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808406069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:26,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808406070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:26,107 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/9f33597a6a5f486cb4e4c5a935126c9e 2024-11-17T01:52:26,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/4e34bf769939445080614decda4f761c is 50, key is test_row_0/C:col10/1731808343873/Put/seqid=0 2024-11-17T01:52:26,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742006_1182 (size=12151) 2024-11-17T01:52:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T01:52:26,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808406174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:26,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808406174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808406174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:26,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808406384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:26,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808406385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:26,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:26,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808406385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:26,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:26,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:26,447 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/41cd3899f765425b92a8c73d2ed5d4c7 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/41cd3899f765425b92a8c73d2ed5d4c7 2024-11-17T01:52:26,452 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/C of 619e0c9a5ad3ede5c5b6d8d44658cb1a into 41cd3899f765425b92a8c73d2ed5d4c7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:26,452 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:26,452 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/C, priority=13, startTime=1731808345579; duration=0sec 2024-11-17T01:52:26,452 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:26,452 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:C 2024-11-17T01:52:26,519 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/4e34bf769939445080614decda4f761c 2024-11-17T01:52:26,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/a1705ce3a47342228f821ffffa0e11e1 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/a1705ce3a47342228f821ffffa0e11e1 2024-11-17T01:52:26,527 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/a1705ce3a47342228f821ffffa0e11e1, entries=150, sequenceid=190, filesize=30.4 K 2024-11-17T01:52:26,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/9f33597a6a5f486cb4e4c5a935126c9e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/9f33597a6a5f486cb4e4c5a935126c9e 2024-11-17T01:52:26,533 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/9f33597a6a5f486cb4e4c5a935126c9e, entries=150, sequenceid=190, filesize=11.9 K 2024-11-17T01:52:26,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/4e34bf769939445080614decda4f761c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/4e34bf769939445080614decda4f761c 2024-11-17T01:52:26,538 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/4e34bf769939445080614decda4f761c, entries=150, sequenceid=190, filesize=11.9 K 2024-11-17T01:52:26,539 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 619e0c9a5ad3ede5c5b6d8d44658cb1a in 879ms, sequenceid=190, compaction requested=false 2024-11-17T01:52:26,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:26,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:26,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-17T01:52:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-17T01:52:26,542 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-17T01:52:26,542 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5630 sec 2024-11-17T01:52:26,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.5680 sec 2024-11-17T01:52:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:26,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-17T01:52:26,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=A 2024-11-17T01:52:26,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:52:26,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=B 2024-11-17T01:52:26,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:52:26,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=C 2024-11-17T01:52:26,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:52:26,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170ec0bfe33cd243cd934705489112ec9e_619e0c9a5ad3ede5c5b6d8d44658cb1a is 50, key is test_row_0/A:col10/1731808346694/Put/seqid=0 2024-11-17T01:52:26,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742007_1183 (size=14794) 2024-11-17T01:52:26,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808406706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:26,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808406707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:26,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808406708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:26,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808406814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:26,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808406823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:26,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808406824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:27,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808407024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:27,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:27,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:27,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:27,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808407034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:27,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808407034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:27,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:27,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:27,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:27,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:27,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:27,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:27,111 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:27,114 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170ec0bfe33cd243cd934705489112ec9e_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170ec0bfe33cd243cd934705489112ec9e_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:27,115 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/28c79e82cf75476fa02a991ee2ce4746, store: [table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:27,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/28c79e82cf75476fa02a991ee2ce4746 is 175, key is test_row_0/A:col10/1731808346694/Put/seqid=0 2024-11-17T01:52:27,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742008_1184 (size=39749) 2024-11-17T01:52:27,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808407334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:27,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808407344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:27,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:27,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:27,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808407344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:27,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:27,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:27,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:27,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:27,520 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/28c79e82cf75476fa02a991ee2ce4746 2024-11-17T01:52:27,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/3007b7ae55a64bfc9fc180235abf0a45 is 50, key is test_row_0/B:col10/1731808346694/Put/seqid=0 2024-11-17T01:52:27,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742009_1185 (size=12151) 2024-11-17T01:52:27,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/3007b7ae55a64bfc9fc180235abf0a45 2024-11-17T01:52:27,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/4b8206b92899475ea92b2ae28956c429 is 50, key is test_row_0/C:col10/1731808346694/Put/seqid=0 2024-11-17T01:52:27,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742010_1186 (size=12151) 2024-11-17T01:52:27,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/4b8206b92899475ea92b2ae28956c429 2024-11-17T01:52:27,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/28c79e82cf75476fa02a991ee2ce4746 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/28c79e82cf75476fa02a991ee2ce4746 2024-11-17T01:52:27,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/28c79e82cf75476fa02a991ee2ce4746, entries=200, sequenceid=214, filesize=38.8 K 2024-11-17T01:52:27,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/3007b7ae55a64bfc9fc180235abf0a45 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/3007b7ae55a64bfc9fc180235abf0a45 2024-11-17T01:52:27,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/3007b7ae55a64bfc9fc180235abf0a45, entries=150, sequenceid=214, filesize=11.9 K 2024-11-17T01:52:27,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/4b8206b92899475ea92b2ae28956c429 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/4b8206b92899475ea92b2ae28956c429 2024-11-17T01:52:27,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/4b8206b92899475ea92b2ae28956c429, entries=150, sequenceid=214, filesize=11.9 K 2024-11-17T01:52:27,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 619e0c9a5ad3ede5c5b6d8d44658cb1a in 956ms, sequenceid=214, compaction requested=true 2024-11-17T01:52:27,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:27,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:52:27,652 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:27,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:27,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:52:27,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:27,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:52:27,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T01:52:27,653 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:27,657 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:27,657 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/A is initiating minor compaction (all files) 2024-11-17T01:52:27,657 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/A in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:27,657 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/4393d9ef630d402a8b475a2212912ce3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/a1705ce3a47342228f821ffffa0e11e1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/28c79e82cf75476fa02a991ee2ce4746] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=100.0 K 2024-11-17T01:52:27,657 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:27,657 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/4393d9ef630d402a8b475a2212912ce3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/a1705ce3a47342228f821ffffa0e11e1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/28c79e82cf75476fa02a991ee2ce4746] 2024-11-17T01:52:27,658 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4393d9ef630d402a8b475a2212912ce3, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731808343222 2024-11-17T01:52:27,658 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:27,658 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/B is initiating minor compaction (all files) 2024-11-17T01:52:27,658 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/B in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:27,658 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/3767cfc658c94b49a49d499cbb512ad3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/9f33597a6a5f486cb4e4c5a935126c9e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/3007b7ae55a64bfc9fc180235abf0a45] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=36.0 K 2024-11-17T01:52:27,661 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 3767cfc658c94b49a49d499cbb512ad3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731808343222 2024-11-17T01:52:27,661 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1705ce3a47342228f821ffffa0e11e1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731808343873 2024-11-17T01:52:27,662 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f33597a6a5f486cb4e4c5a935126c9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731808343873 2024-11-17T01:52:27,663 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28c79e82cf75476fa02a991ee2ce4746, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731808346065 2024-11-17T01:52:27,663 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 3007b7ae55a64bfc9fc180235abf0a45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731808346067 2024-11-17T01:52:27,691 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:27,693 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#B#compaction#157 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:27,694 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/25a64f771f9942bd92393c01f2a8414d is 50, key is test_row_0/B:col10/1731808346694/Put/seqid=0 2024-11-17T01:52:27,706 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117a6699704221b470fa61faa626c757f29_619e0c9a5ad3ede5c5b6d8d44658cb1a store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:27,708 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117a6699704221b470fa61faa626c757f29_619e0c9a5ad3ede5c5b6d8d44658cb1a, store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:27,708 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a6699704221b470fa61faa626c757f29_619e0c9a5ad3ede5c5b6d8d44658cb1a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:27,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742012_1188 (size=4469) 2024-11-17T01:52:27,716 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#A#compaction#156 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:27,717 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/0ceddfa7ed0e40a38e66b0e908560c5d is 175, key is test_row_0/A:col10/1731808346694/Put/seqid=0 2024-11-17T01:52:27,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742011_1187 (size=12663) 2024-11-17T01:52:27,740 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/25a64f771f9942bd92393c01f2a8414d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/25a64f771f9942bd92393c01f2a8414d 2024-11-17T01:52:27,752 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/B of 619e0c9a5ad3ede5c5b6d8d44658cb1a into 25a64f771f9942bd92393c01f2a8414d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:27,752 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:27,752 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/B, priority=13, startTime=1731808347652; duration=0sec 2024-11-17T01:52:27,752 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:27,752 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:B 2024-11-17T01:52:27,752 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:27,753 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:27,753 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/C is initiating minor compaction (all files) 2024-11-17T01:52:27,753 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/C in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:27,754 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/41cd3899f765425b92a8c73d2ed5d4c7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/4e34bf769939445080614decda4f761c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/4b8206b92899475ea92b2ae28956c429] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=36.0 K 2024-11-17T01:52:27,754 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 41cd3899f765425b92a8c73d2ed5d4c7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731808343222 2024-11-17T01:52:27,755 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e34bf769939445080614decda4f761c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731808343873 2024-11-17T01:52:27,755 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b8206b92899475ea92b2ae28956c429, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731808346067 2024-11-17T01:52:27,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742013_1189 (size=31617) 2024-11-17T01:52:27,771 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#C#compaction#158 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:27,772 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/b01c9b4a64824cd2869de4a9bbc64d4f is 50, key is test_row_0/C:col10/1731808346694/Put/seqid=0 2024-11-17T01:52:27,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742014_1190 (size=12663) 2024-11-17T01:52:27,795 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/b01c9b4a64824cd2869de4a9bbc64d4f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/b01c9b4a64824cd2869de4a9bbc64d4f 2024-11-17T01:52:27,807 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/C of 619e0c9a5ad3ede5c5b6d8d44658cb1a into b01c9b4a64824cd2869de4a9bbc64d4f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:27,807 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:27,807 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/C, priority=13, startTime=1731808347652; duration=0sec 2024-11-17T01:52:27,807 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:27,808 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:C 2024-11-17T01:52:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:27,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T01:52:27,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=A 2024-11-17T01:52:27,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:27,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=B 2024-11-17T01:52:27,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:27,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=C 2024-11-17T01:52:27,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:27,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d59bb25d5354419c85d6654e9ac59743_619e0c9a5ad3ede5c5b6d8d44658cb1a is 50, key is test_row_0/A:col10/1731808346699/Put/seqid=0 2024-11-17T01:52:27,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808407894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:27,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:27,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808407896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:27,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:27,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:27,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:27,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:27,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:27,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808407903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:27,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:27,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:27,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:27,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742015_1191 (size=14794) 2024-11-17T01:52:28,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808408014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:28,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808408014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:28,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808408014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T01:52:28,135 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:28,135 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:52:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-17T01:52:28,137 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:52:28,138 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:52:28,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:52:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T01:52:28,171 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/0ceddfa7ed0e40a38e66b0e908560c5d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/0ceddfa7ed0e40a38e66b0e908560c5d 2024-11-17T01:52:28,185 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/A of 619e0c9a5ad3ede5c5b6d8d44658cb1a into 0ceddfa7ed0e40a38e66b0e908560c5d(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:28,185 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:28,185 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/A, priority=13, startTime=1731808347652; duration=0sec 2024-11-17T01:52:28,185 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:28,185 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:A 2024-11-17T01:52:28,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808408224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:28,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:28,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808408224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:28,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:28,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:28,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:28,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:28,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:28,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808408224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T01:52:28,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T01:52:28,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:28,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:28,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:28,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:28,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:28,314 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:28,323 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d59bb25d5354419c85d6654e9ac59743_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d59bb25d5354419c85d6654e9ac59743_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:28,324 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/a7bf3cdb216d4b65ac8145d8763de53c, store: [table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:28,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/a7bf3cdb216d4b65ac8145d8763de53c is 175, key is test_row_0/A:col10/1731808346699/Put/seqid=0 2024-11-17T01:52:28,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742016_1192 (size=39749) 2024-11-17T01:52:28,376 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=228, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/a7bf3cdb216d4b65ac8145d8763de53c 2024-11-17T01:52:28,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/5cf84cc104a9463c9ed10c0facf6b532 is 50, key is test_row_0/B:col10/1731808346699/Put/seqid=0 2024-11-17T01:52:28,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742017_1193 (size=12151) 2024-11-17T01:52:28,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T01:52:28,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:28,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:28,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:28,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:28,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T01:52:28,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808408535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:28,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808408545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:28,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808408545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:28,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:28,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:28,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:28,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:28,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:28,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:28,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T01:52:28,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:28,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:28,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:28,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:28,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:28,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T01:52:28,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:28,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:28,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:28,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:28,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T01:52:28,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/5cf84cc104a9463c9ed10c0facf6b532 2024-11-17T01:52:28,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/0ae1333aac544e02830e65c92f979428 is 50, key is test_row_0/C:col10/1731808346699/Put/seqid=0 2024-11-17T01:52:28,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742018_1194 (size=12151) 2024-11-17T01:52:28,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/0ae1333aac544e02830e65c92f979428 2024-11-17T01:52:28,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/a7bf3cdb216d4b65ac8145d8763de53c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/a7bf3cdb216d4b65ac8145d8763de53c 2024-11-17T01:52:28,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/a7bf3cdb216d4b65ac8145d8763de53c, entries=200, sequenceid=228, filesize=38.8 K 2024-11-17T01:52:28,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/5cf84cc104a9463c9ed10c0facf6b532 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/5cf84cc104a9463c9ed10c0facf6b532 2024-11-17T01:52:28,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/5cf84cc104a9463c9ed10c0facf6b532, entries=150, sequenceid=228, filesize=11.9 K 2024-11-17T01:52:28,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/0ae1333aac544e02830e65c92f979428 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/0ae1333aac544e02830e65c92f979428 2024-11-17T01:52:28,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/0ae1333aac544e02830e65c92f979428, entries=150, sequenceid=228, filesize=11.9 K 2024-11-17T01:52:28,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 619e0c9a5ad3ede5c5b6d8d44658cb1a in 1021ms, sequenceid=228, compaction requested=false 2024-11-17T01:52:28,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:28,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T01:52:28,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:28,911 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T01:52:28,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=A 2024-11-17T01:52:28,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:28,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=B 2024-11-17T01:52:28,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:28,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=C 2024-11-17T01:52:28,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:28,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111744393503042e42f7af39630afc020273_619e0c9a5ad3ede5c5b6d8d44658cb1a is 50, key is test_row_0/A:col10/1731808347894/Put/seqid=0 2024-11-17T01:52:28,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742019_1195 (size=12304) 2024-11-17T01:52:28,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:28,947 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111744393503042e42f7af39630afc020273_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111744393503042e42f7af39630afc020273_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:28,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/ec9910abc0ad4e6a8e6fd3a8e1d19be7, store: [table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:28,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/ec9910abc0ad4e6a8e6fd3a8e1d19be7 is 175, key is test_row_0/A:col10/1731808347894/Put/seqid=0 2024-11-17T01:52:28,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742020_1196 (size=31105) 2024-11-17T01:52:28,965 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/ec9910abc0ad4e6a8e6fd3a8e1d19be7 2024-11-17T01:52:28,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/874f188651ec4b418a33f004c8090d35 is 50, key is test_row_0/B:col10/1731808347894/Put/seqid=0 2024-11-17T01:52:28,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742021_1197 (size=12151) 2024-11-17T01:52:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:29,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:29,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808409066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:29,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808409066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808409067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:29,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808409184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808409184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808409184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T01:52:29,393 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/874f188651ec4b418a33f004c8090d35 2024-11-17T01:52:29,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808409394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:29,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808409394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:29,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:29,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808409394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:29,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:29,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:29,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/51b6a3ab67504c9387e4c9847771f2a9 is 50, key is test_row_0/C:col10/1731808347894/Put/seqid=0 2024-11-17T01:52:29,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742022_1198 (size=12151) 2024-11-17T01:52:29,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808409704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:29,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808409704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:29,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:29,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808409704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:29,809 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/51b6a3ab67504c9387e4c9847771f2a9 2024-11-17T01:52:29,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/ec9910abc0ad4e6a8e6fd3a8e1d19be7 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/ec9910abc0ad4e6a8e6fd3a8e1d19be7 2024-11-17T01:52:29,821 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/ec9910abc0ad4e6a8e6fd3a8e1d19be7, entries=150, sequenceid=249, filesize=30.4 K 2024-11-17T01:52:29,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/874f188651ec4b418a33f004c8090d35 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/874f188651ec4b418a33f004c8090d35 2024-11-17T01:52:29,827 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/874f188651ec4b418a33f004c8090d35, entries=150, sequenceid=249, filesize=11.9 K 2024-11-17T01:52:29,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/51b6a3ab67504c9387e4c9847771f2a9 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/51b6a3ab67504c9387e4c9847771f2a9 2024-11-17T01:52:29,834 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/51b6a3ab67504c9387e4c9847771f2a9, entries=150, sequenceid=249, filesize=11.9 K 2024-11-17T01:52:29,835 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 619e0c9a5ad3ede5c5b6d8d44658cb1a in 924ms, sequenceid=249, compaction requested=true 2024-11-17T01:52:29,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:29,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:29,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-17T01:52:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-17T01:52:29,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-17T01:52:29,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6990 sec 2024-11-17T01:52:29,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.7060 sec 2024-11-17T01:52:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:30,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T01:52:30,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=A 2024-11-17T01:52:30,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:30,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=B 2024-11-17T01:52:30,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:30,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=C 2024-11-17T01:52:30,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:30,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111704437e0471d04fa990e6c00d42bd9a76_619e0c9a5ad3ede5c5b6d8d44658cb1a is 50, key is test_row_0/A:col10/1731808349063/Put/seqid=0 2024-11-17T01:52:30,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:30,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808410242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:30,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808410242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:30,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:30,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808410243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:30,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:30,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742023_1199 (size=14994) 2024-11-17T01:52:30,249 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:30,252 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111704437e0471d04fa990e6c00d42bd9a76_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111704437e0471d04fa990e6c00d42bd9a76_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:30,253 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/8010395cf75e4712bff97c68765af222, store: [table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:30,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/8010395cf75e4712bff97c68765af222 is 175, key is test_row_0/A:col10/1731808349063/Put/seqid=0 2024-11-17T01:52:30,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742024_1200 (size=39949) 2024-11-17T01:52:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T01:52:30,285 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:30,285 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:52:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-17T01:52:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T01:52:30,286 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:52:30,287 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:52:30,287 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:52:30,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808410354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:30,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808410354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:30,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:30,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808410354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:30,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:30,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T01:52:30,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T01:52:30,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:30,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:30,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:30,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:30,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:30,502 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T01:52:30,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:30,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808410564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:30,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808410564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:30,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808410564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:30,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T01:52:30,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:30,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:30,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:30,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:30,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T01:52:30,657 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/8010395cf75e4712bff97c68765af222 2024-11-17T01:52:30,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/7b886d9af67444ecb1d6ee052e1f60b2 is 50, key is test_row_0/B:col10/1731808349063/Put/seqid=0 2024-11-17T01:52:30,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742025_1201 (size=12301) 2024-11-17T01:52:30,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T01:52:30,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:30,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:30,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:30,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:30,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:30,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808410874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:30,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808410874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:30,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808410874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:30,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:30,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T01:52:30,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:30,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:30,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:30,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:30,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T01:52:31,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T01:52:31,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:31,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:31,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:31,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:31,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:31,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/7b886d9af67444ecb1d6ee052e1f60b2 2024-11-17T01:52:31,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/a028e2a0082d42b5aa5f4a84db2dda2f is 50, key is test_row_0/C:col10/1731808349063/Put/seqid=0 2024-11-17T01:52:31,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742026_1202 (size=12301) 2024-11-17T01:52:31,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/a028e2a0082d42b5aa5f4a84db2dda2f 2024-11-17T01:52:31,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/8010395cf75e4712bff97c68765af222 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/8010395cf75e4712bff97c68765af222 2024-11-17T01:52:31,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/8010395cf75e4712bff97c68765af222, entries=200, sequenceid=263, filesize=39.0 K 2024-11-17T01:52:31,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/7b886d9af67444ecb1d6ee052e1f60b2 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/7b886d9af67444ecb1d6ee052e1f60b2 2024-11-17T01:52:31,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/7b886d9af67444ecb1d6ee052e1f60b2, entries=150, sequenceid=263, filesize=12.0 K 2024-11-17T01:52:31,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/a028e2a0082d42b5aa5f4a84db2dda2f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/a028e2a0082d42b5aa5f4a84db2dda2f 2024-11-17T01:52:31,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/a028e2a0082d42b5aa5f4a84db2dda2f, entries=150, sequenceid=263, filesize=12.0 K 2024-11-17T01:52:31,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 619e0c9a5ad3ede5c5b6d8d44658cb1a in 913ms, sequenceid=263, compaction requested=true 2024-11-17T01:52:31,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:31,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:52:31,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:31,129 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:52:31,129 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:52:31,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:52:31,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:31,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:52:31,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:31,131 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142420 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:52:31,131 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/A is initiating minor compaction (all files) 2024-11-17T01:52:31,131 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/A in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:31,131 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/0ceddfa7ed0e40a38e66b0e908560c5d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/a7bf3cdb216d4b65ac8145d8763de53c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/ec9910abc0ad4e6a8e6fd3a8e1d19be7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/8010395cf75e4712bff97c68765af222] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=139.1 K 2024-11-17T01:52:31,131 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:31,131 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/0ceddfa7ed0e40a38e66b0e908560c5d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/a7bf3cdb216d4b65ac8145d8763de53c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/ec9910abc0ad4e6a8e6fd3a8e1d19be7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/8010395cf75e4712bff97c68765af222] 2024-11-17T01:52:31,132 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ceddfa7ed0e40a38e66b0e908560c5d, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731808346067 2024-11-17T01:52:31,132 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:52:31,132 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/B is initiating minor compaction (all files) 2024-11-17T01:52:31,132 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/B in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:31,132 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7bf3cdb216d4b65ac8145d8763de53c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731808346699 2024-11-17T01:52:31,132 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/25a64f771f9942bd92393c01f2a8414d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/5cf84cc104a9463c9ed10c0facf6b532, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/874f188651ec4b418a33f004c8090d35, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/7b886d9af67444ecb1d6ee052e1f60b2] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=48.1 K 2024-11-17T01:52:31,133 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec9910abc0ad4e6a8e6fd3a8e1d19be7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731808347892 2024-11-17T01:52:31,133 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 25a64f771f9942bd92393c01f2a8414d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731808346067 2024-11-17T01:52:31,133 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8010395cf75e4712bff97c68765af222, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731808349062 2024-11-17T01:52:31,134 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cf84cc104a9463c9ed10c0facf6b532, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731808346699 2024-11-17T01:52:31,134 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 874f188651ec4b418a33f004c8090d35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731808347892 2024-11-17T01:52:31,134 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b886d9af67444ecb1d6ee052e1f60b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731808349063 2024-11-17T01:52:31,146 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#B#compaction#168 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:31,147 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/4d197e9e1b1440a28aaf560cface9f42 is 50, key is test_row_0/B:col10/1731808349063/Put/seqid=0 2024-11-17T01:52:31,157 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:31,174 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111775aba644fd474d82926e7ce45907b07c_619e0c9a5ad3ede5c5b6d8d44658cb1a store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:31,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742027_1203 (size=12949) 2024-11-17T01:52:31,177 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111775aba644fd474d82926e7ce45907b07c_619e0c9a5ad3ede5c5b6d8d44658cb1a, store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:31,177 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111775aba644fd474d82926e7ce45907b07c_619e0c9a5ad3ede5c5b6d8d44658cb1a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:31,181 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/4d197e9e1b1440a28aaf560cface9f42 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/4d197e9e1b1440a28aaf560cface9f42 2024-11-17T01:52:31,186 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/B of 619e0c9a5ad3ede5c5b6d8d44658cb1a into 4d197e9e1b1440a28aaf560cface9f42(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:31,186 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:31,186 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/B, priority=12, startTime=1731808351129; duration=0sec 2024-11-17T01:52:31,187 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:31,187 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:B 2024-11-17T01:52:31,187 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:52:31,188 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:52:31,188 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/C is initiating minor compaction (all files) 2024-11-17T01:52:31,188 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/C in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:31,188 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/b01c9b4a64824cd2869de4a9bbc64d4f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/0ae1333aac544e02830e65c92f979428, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/51b6a3ab67504c9387e4c9847771f2a9, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/a028e2a0082d42b5aa5f4a84db2dda2f] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=48.1 K 2024-11-17T01:52:31,188 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting b01c9b4a64824cd2869de4a9bbc64d4f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731808346067 2024-11-17T01:52:31,189 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ae1333aac544e02830e65c92f979428, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731808346699 2024-11-17T01:52:31,189 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 51b6a3ab67504c9387e4c9847771f2a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731808347892 2024-11-17T01:52:31,189 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting a028e2a0082d42b5aa5f4a84db2dda2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731808349063 2024-11-17T01:52:31,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T01:52:31,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:31,204 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T01:52:31,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=A 2024-11-17T01:52:31,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:31,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=B 2024-11-17T01:52:31,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:31,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=C 2024-11-17T01:52:31,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:31,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742028_1204 (size=4469) 2024-11-17T01:52:31,221 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#C#compaction#170 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:31,221 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/5a066c6f08cf43c9b7634694b3136dfd is 50, key is test_row_0/C:col10/1731808349063/Put/seqid=0 2024-11-17T01:52:31,222 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#A#compaction#169 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:31,223 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/6425720c792f4d299432ae88ab3e78db is 175, key is test_row_0/A:col10/1731808349063/Put/seqid=0 2024-11-17T01:52:31,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174470583b1b2d4454b2a71fdc8c6197ad_619e0c9a5ad3ede5c5b6d8d44658cb1a is 50, key is test_row_0/A:col10/1731808350239/Put/seqid=0 2024-11-17T01:52:31,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742030_1206 (size=12949) 2024-11-17T01:52:31,251 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/5a066c6f08cf43c9b7634694b3136dfd as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/5a066c6f08cf43c9b7634694b3136dfd 2024-11-17T01:52:31,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742029_1205 (size=31903) 2024-11-17T01:52:31,262 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/C of 619e0c9a5ad3ede5c5b6d8d44658cb1a into 5a066c6f08cf43c9b7634694b3136dfd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:31,262 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:31,262 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/C, priority=12, startTime=1731808351129; duration=0sec 2024-11-17T01:52:31,262 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:31,262 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:C 2024-11-17T01:52:31,270 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/6425720c792f4d299432ae88ab3e78db as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/6425720c792f4d299432ae88ab3e78db 2024-11-17T01:52:31,278 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/A of 619e0c9a5ad3ede5c5b6d8d44658cb1a into 6425720c792f4d299432ae88ab3e78db(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:31,278 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:31,278 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/A, priority=12, startTime=1731808351129; duration=0sec 2024-11-17T01:52:31,278 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:31,278 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:A 2024-11-17T01:52:31,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742031_1207 (size=12454) 2024-11-17T01:52:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:31,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. as already flushing 2024-11-17T01:52:31,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808411403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:31,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808411403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:31,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808411404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T01:52:31,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808411514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:31,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808411514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808411514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:31,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:31,684 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174470583b1b2d4454b2a71fdc8c6197ad_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174470583b1b2d4454b2a71fdc8c6197ad_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:31,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/719c1742141e4ad99d6264b659bb5f77, store: [table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:31,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/719c1742141e4ad99d6264b659bb5f77 is 175, key is test_row_0/A:col10/1731808350239/Put/seqid=0 2024-11-17T01:52:31,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742032_1208 (size=31255) 2024-11-17T01:52:31,690 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/719c1742141e4ad99d6264b659bb5f77 2024-11-17T01:52:31,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/f95f94a4065a4036960660e8d62966b2 is 50, key is test_row_0/B:col10/1731808350239/Put/seqid=0 2024-11-17T01:52:31,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742033_1209 (size=12301) 2024-11-17T01:52:31,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:31,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808411724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808411724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:31,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808411724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:31,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:31,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51474 deadline: 1731808411874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:31,877 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_2' in 619e0c9a5ad3ede5c5b6d8d44658cb1a of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:52:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:31,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:31,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51480 deadline: 1731808411914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:31,917 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in 619e0c9a5ad3ede5c5b6d8d44658cb1a of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18297 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:52:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:31,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:32,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:32,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808412034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:32,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:32,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808412034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:32,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:32,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808412034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:32,120 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/f95f94a4065a4036960660e8d62966b2 2024-11-17T01:52:32,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/d631f84f5b1a4ae1bde6ba19bfa0ccef is 50, key is test_row_0/C:col10/1731808350239/Put/seqid=0 2024-11-17T01:52:32,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742034_1210 (size=12301) 2024-11-17T01:52:32,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T01:52:32,532 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/d631f84f5b1a4ae1bde6ba19bfa0ccef 2024-11-17T01:52:32,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/719c1742141e4ad99d6264b659bb5f77 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/719c1742141e4ad99d6264b659bb5f77 2024-11-17T01:52:32,540 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/719c1742141e4ad99d6264b659bb5f77, entries=150, sequenceid=282, filesize=30.5 K 2024-11-17T01:52:32,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/f95f94a4065a4036960660e8d62966b2 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/f95f94a4065a4036960660e8d62966b2 2024-11-17T01:52:32,545 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/f95f94a4065a4036960660e8d62966b2, entries=150, sequenceid=282, filesize=12.0 K 2024-11-17T01:52:32,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/d631f84f5b1a4ae1bde6ba19bfa0ccef as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/d631f84f5b1a4ae1bde6ba19bfa0ccef 2024-11-17T01:52:32,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:32,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51424 deadline: 1731808412544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:32,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:32,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:32,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:32,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:32,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51466 deadline: 1731808412544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:32,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:32,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:32,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:32,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:32,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1731808412545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:32,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:32,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:32,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:52:32,551 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/d631f84f5b1a4ae1bde6ba19bfa0ccef, entries=150, sequenceid=282, filesize=12.0 K 2024-11-17T01:52:32,552 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 619e0c9a5ad3ede5c5b6d8d44658cb1a in 1349ms, sequenceid=282, compaction requested=false 2024-11-17T01:52:32,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:32,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:32,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-17T01:52:32,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-17T01:52:32,555 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-17T01:52:32,555 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2660 sec 2024-11-17T01:52:32,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.2700 sec 2024-11-17T01:52:33,498 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-11-17T01:52:33,498 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:52:33,498 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:33,498 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:33,498 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:52:33,501 INFO [Thread-646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-646. 2024-11-17T01:52:33,501 DEBUG [Thread-646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:52:33,501 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:33,501 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:33,501 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:52:33,502 INFO [Thread-652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-652. 2024-11-17T01:52:33,502 DEBUG [Thread-652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:52:33,502 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:33,502 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:33,502 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:52:33,504 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-17T01:52:33,504 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:52:33,504 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:33,504 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:33,504 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:52:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:33,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T01:52:33,556 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-17T01:52:33,557 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:52:33,557 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:33,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=A 2024-11-17T01:52:33,557 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:33,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:33,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=B 2024-11-17T01:52:33,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:33,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=C 2024-11-17T01:52:33,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:33,557 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:52:33,557 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-17T01:52:33,557 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:52:33,557 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:33,557 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:33,557 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:52:33,565 INFO [Thread-641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-641. 2024-11-17T01:52:33,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117df2e04c144f34bff930da8e9b8506b75_619e0c9a5ad3ede5c5b6d8d44658cb1a is 50, key is test_row_0/A:col10/1731808353555/Put/seqid=0 2024-11-17T01:52:33,565 DEBUG [Thread-641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:52:33,565 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:33,565 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:33,565 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:52:33,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742035_1211 (size=12454) 2024-11-17T01:52:33,973 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:33,979 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117df2e04c144f34bff930da8e9b8506b75_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117df2e04c144f34bff930da8e9b8506b75_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:33,980 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/7f950f6bac704307a8d9d5f269bd2a3e, store: [table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:33,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/7f950f6bac704307a8d9d5f269bd2a3e is 175, key is test_row_0/A:col10/1731808353555/Put/seqid=0 2024-11-17T01:52:33,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742036_1212 (size=31255) 2024-11-17T01:52:34,388 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=301, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/7f950f6bac704307a8d9d5f269bd2a3e 2024-11-17T01:52:34,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/4b8026cb25f4474a8a2a817a8846cf94 is 50, key is test_row_0/B:col10/1731808353555/Put/seqid=0 2024-11-17T01:52:34,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742037_1213 (size=12301) 2024-11-17T01:52:34,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T01:52:34,455 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:34,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/4b8026cb25f4474a8a2a817a8846cf94 2024-11-17T01:52:34,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/6614810f187b497181d815fcc761b812 is 50, key is test_row_0/C:col10/1731808353555/Put/seqid=0 2024-11-17T01:52:34,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742038_1214 (size=12301) 2024-11-17T01:52:35,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/6614810f187b497181d815fcc761b812 2024-11-17T01:52:35,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/7f950f6bac704307a8d9d5f269bd2a3e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/7f950f6bac704307a8d9d5f269bd2a3e 2024-11-17T01:52:35,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/7f950f6bac704307a8d9d5f269bd2a3e, entries=150, sequenceid=301, filesize=30.5 K 2024-11-17T01:52:35,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/4b8026cb25f4474a8a2a817a8846cf94 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/4b8026cb25f4474a8a2a817a8846cf94 2024-11-17T01:52:35,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/4b8026cb25f4474a8a2a817a8846cf94, entries=150, sequenceid=301, filesize=12.0 K 2024-11-17T01:52:35,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/6614810f187b497181d815fcc761b812 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/6614810f187b497181d815fcc761b812 2024-11-17T01:52:35,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/6614810f187b497181d815fcc761b812, entries=150, sequenceid=301, filesize=12.0 K 2024-11-17T01:52:35,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=6.71 KB/6870 for 619e0c9a5ad3ede5c5b6d8d44658cb1a in 1694ms, sequenceid=301, compaction requested=true 2024-11-17T01:52:35,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:35,251 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:35,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:52:35,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:35,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:52:35,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:35,251 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:35,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619e0c9a5ad3ede5c5b6d8d44658cb1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:52:35,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:35,252 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:35,252 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/A is initiating minor compaction (all files) 2024-11-17T01:52:35,252 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/A in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:35,253 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/6425720c792f4d299432ae88ab3e78db, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/719c1742141e4ad99d6264b659bb5f77, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/7f950f6bac704307a8d9d5f269bd2a3e] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=92.2 K 2024-11-17T01:52:35,253 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:35,253 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/6425720c792f4d299432ae88ab3e78db, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/719c1742141e4ad99d6264b659bb5f77, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/7f950f6bac704307a8d9d5f269bd2a3e] 2024-11-17T01:52:35,253 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6425720c792f4d299432ae88ab3e78db, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731808349063 2024-11-17T01:52:35,254 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:35,254 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 719c1742141e4ad99d6264b659bb5f77, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731808350224 2024-11-17T01:52:35,254 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/B is initiating minor compaction (all files) 2024-11-17T01:52:35,254 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/B in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:35,254 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/4d197e9e1b1440a28aaf560cface9f42, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/f95f94a4065a4036960660e8d62966b2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/4b8026cb25f4474a8a2a817a8846cf94] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=36.7 K 2024-11-17T01:52:35,254 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f950f6bac704307a8d9d5f269bd2a3e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731808351401 2024-11-17T01:52:35,255 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d197e9e1b1440a28aaf560cface9f42, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731808349063 2024-11-17T01:52:35,256 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting f95f94a4065a4036960660e8d62966b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731808350224 2024-11-17T01:52:35,256 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b8026cb25f4474a8a2a817a8846cf94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731808351401 2024-11-17T01:52:35,278 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:35,283 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#B#compaction#177 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:35,283 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/8c61b72d933843b59e0f8b7377f52539 is 50, key is test_row_0/B:col10/1731808353555/Put/seqid=0 2024-11-17T01:52:35,284 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111709916ce4f2d7400e9662905e42eba89d_619e0c9a5ad3ede5c5b6d8d44658cb1a store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:35,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742039_1215 (size=13051) 2024-11-17T01:52:35,310 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/8c61b72d933843b59e0f8b7377f52539 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/8c61b72d933843b59e0f8b7377f52539 2024-11-17T01:52:35,318 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/B of 619e0c9a5ad3ede5c5b6d8d44658cb1a into 8c61b72d933843b59e0f8b7377f52539(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:35,318 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:35,318 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/B, priority=13, startTime=1731808355251; duration=0sec 2024-11-17T01:52:35,318 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:35,318 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:B 2024-11-17T01:52:35,318 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:35,320 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:35,320 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 619e0c9a5ad3ede5c5b6d8d44658cb1a/C is initiating minor compaction (all files) 2024-11-17T01:52:35,320 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619e0c9a5ad3ede5c5b6d8d44658cb1a/C in TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:35,320 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/5a066c6f08cf43c9b7634694b3136dfd, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/d631f84f5b1a4ae1bde6ba19bfa0ccef, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/6614810f187b497181d815fcc761b812] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp, totalSize=36.7 K 2024-11-17T01:52:35,321 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a066c6f08cf43c9b7634694b3136dfd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731808349063 2024-11-17T01:52:35,321 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting d631f84f5b1a4ae1bde6ba19bfa0ccef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731808350224 2024-11-17T01:52:35,322 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 6614810f187b497181d815fcc761b812, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731808351401 2024-11-17T01:52:35,348 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#C#compaction#179 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:35,348 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111709916ce4f2d7400e9662905e42eba89d_619e0c9a5ad3ede5c5b6d8d44658cb1a, store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:35,348 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111709916ce4f2d7400e9662905e42eba89d_619e0c9a5ad3ede5c5b6d8d44658cb1a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:35,348 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/96a3ab5af25a41208d1cda3971859414 is 50, key is test_row_0/C:col10/1731808353555/Put/seqid=0 2024-11-17T01:52:35,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742041_1217 (size=13051) 2024-11-17T01:52:35,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742040_1216 (size=4469) 2024-11-17T01:52:35,798 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619e0c9a5ad3ede5c5b6d8d44658cb1a#A#compaction#178 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:35,799 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/057210b8586f4a1da07b2ff1a86685d7 is 175, key is test_row_0/A:col10/1731808353555/Put/seqid=0 2024-11-17T01:52:35,810 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/96a3ab5af25a41208d1cda3971859414 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/96a3ab5af25a41208d1cda3971859414 2024-11-17T01:52:35,816 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/C of 619e0c9a5ad3ede5c5b6d8d44658cb1a into 96a3ab5af25a41208d1cda3971859414(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:35,816 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:35,816 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/C, priority=13, startTime=1731808355251; duration=0sec 2024-11-17T01:52:35,816 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:35,816 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:C 2024-11-17T01:52:35,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742042_1218 (size=32005) 2024-11-17T01:52:35,830 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/057210b8586f4a1da07b2ff1a86685d7 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/057210b8586f4a1da07b2ff1a86685d7 2024-11-17T01:52:35,839 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619e0c9a5ad3ede5c5b6d8d44658cb1a/A of 619e0c9a5ad3ede5c5b6d8d44658cb1a into 057210b8586f4a1da07b2ff1a86685d7(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:35,840 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: 2024-11-17T01:52:35,840 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., storeName=619e0c9a5ad3ede5c5b6d8d44658cb1a/A, priority=13, startTime=1731808355251; duration=0sec 2024-11-17T01:52:35,840 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:35,840 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619e0c9a5ad3ede5c5b6d8d44658cb1a:A 2024-11-17T01:52:41,916 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-17T01:52:41,916 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:52:41,916 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:41,916 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:41,916 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:52:42,005 INFO [Thread-643 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-643. 2024-11-17T01:52:42,005 DEBUG [Thread-643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:52:42,005 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:42,005 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:42,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:52:42,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T01:52:42,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 6 2024-11-17T01:52:42,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-17T01:52:42,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-17T01:52:42,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-17T01:52:42,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 5 2024-11-17T01:52:42,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T01:52:42,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3345 2024-11-17T01:52:42,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3464 2024-11-17T01:52:42,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T01:52:42,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1448 2024-11-17T01:52:42,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4337 rows 2024-11-17T01:52:42,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1455 2024-11-17T01:52:42,006 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4344 rows 2024-11-17T01:52:42,006 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T01:52:42,006 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:52:42,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:42,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:42,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:52:42,007 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T01:52:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T01:52:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T01:52:42,011 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808362010"}]},"ts":"1731808362010"} 2024-11-17T01:52:42,012 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T01:52:42,012 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T01:52:42,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T01:52:42,015 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619e0c9a5ad3ede5c5b6d8d44658cb1a, UNASSIGN}] 2024-11-17T01:52:42,016 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619e0c9a5ad3ede5c5b6d8d44658cb1a, UNASSIGN 2024-11-17T01:52:42,017 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=619e0c9a5ad3ede5c5b6d8d44658cb1a, regionState=CLOSING, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:42,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=619e0c9a5ad3ede5c5b6d8d44658cb1a, UNASSIGN because future has completed 2024-11-17T01:52:42,019 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T01:52:42,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173}] 2024-11-17T01:52:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T01:52:42,172 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:42,172 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T01:52:42,172 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 619e0c9a5ad3ede5c5b6d8d44658cb1a, disabling compactions & flushes 2024-11-17T01:52:42,172 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:42,172 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:42,172 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. after waiting 0 ms 2024-11-17T01:52:42,172 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:42,172 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-17T01:52:42,173 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=A 2024-11-17T01:52:42,173 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:42,173 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=B 2024-11-17T01:52:42,173 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:42,173 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619e0c9a5ad3ede5c5b6d8d44658cb1a, store=C 2024-11-17T01:52:42,173 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:42,180 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177d4e8e038e944628a722a11ca4fe81ae_619e0c9a5ad3ede5c5b6d8d44658cb1a is 50, key is test_row_0/A:col10/1731808353564/Put/seqid=0 2024-11-17T01:52:42,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742043_1219 (size=12454) 2024-11-17T01:52:42,189 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:42,195 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177d4e8e038e944628a722a11ca4fe81ae_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177d4e8e038e944628a722a11ca4fe81ae_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:42,196 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/7483419f63914f62ba812e1b3a25a3f3, store: [table=TestAcidGuarantees family=A region=619e0c9a5ad3ede5c5b6d8d44658cb1a] 2024-11-17T01:52:42,197 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/7483419f63914f62ba812e1b3a25a3f3 is 175, key is test_row_0/A:col10/1731808353564/Put/seqid=0 2024-11-17T01:52:42,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742044_1220 (size=31255) 2024-11-17T01:52:42,206 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/7483419f63914f62ba812e1b3a25a3f3 2024-11-17T01:52:42,216 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/a23ef7e3e62e4359ad5af46c4abc867e is 50, key is test_row_0/B:col10/1731808353564/Put/seqid=0 2024-11-17T01:52:42,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742045_1221 (size=12301) 2024-11-17T01:52:42,223 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/a23ef7e3e62e4359ad5af46c4abc867e 2024-11-17T01:52:42,232 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/fb8456418fb142fa868f2f026cddceb9 is 50, key is test_row_0/C:col10/1731808353564/Put/seqid=0 2024-11-17T01:52:42,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742046_1222 (size=12301) 2024-11-17T01:52:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T01:52:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T01:52:42,637 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/fb8456418fb142fa868f2f026cddceb9 2024-11-17T01:52:42,643 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/A/7483419f63914f62ba812e1b3a25a3f3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/7483419f63914f62ba812e1b3a25a3f3 2024-11-17T01:52:42,648 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/7483419f63914f62ba812e1b3a25a3f3, entries=150, sequenceid=310, filesize=30.5 K 2024-11-17T01:52:42,649 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/B/a23ef7e3e62e4359ad5af46c4abc867e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a23ef7e3e62e4359ad5af46c4abc867e 2024-11-17T01:52:42,654 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a23ef7e3e62e4359ad5af46c4abc867e, entries=150, sequenceid=310, filesize=12.0 K 2024-11-17T01:52:42,655 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/.tmp/C/fb8456418fb142fa868f2f026cddceb9 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/fb8456418fb142fa868f2f026cddceb9 2024-11-17T01:52:42,660 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/fb8456418fb142fa868f2f026cddceb9, entries=150, sequenceid=310, filesize=12.0 K 2024-11-17T01:52:42,661 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 619e0c9a5ad3ede5c5b6d8d44658cb1a in 489ms, sequenceid=310, compaction requested=false 2024-11-17T01:52:42,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/ec69216d44034c89a85dee7fd3daf631, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/1392b9f817c74415944182303d9c06e0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/7cd0262b733d4a84b4c35bf31f4e3b58, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/b08b643bec984b3e8cf1c09d04fbe3dd, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/0722c4e70eb048ae8223da516af80a63, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/26e4ca2dd1b5442eb50eb608f8c1315e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/f22a4e6372574d10bd154bc4fe2b26a9, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/f67b15549d184fa5ae9503be14c41eb2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/b43903a6a40c4fa18ab46248f131b450, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/6ae8c476c110496a888b2cba9d60b0e2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/fa239086aab547b2ad48d8f9ab9936c4, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/4393d9ef630d402a8b475a2212912ce3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/6a4c9520593342c5b19d79226c9fd6f8, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/a1705ce3a47342228f821ffffa0e11e1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/28c79e82cf75476fa02a991ee2ce4746, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/0ceddfa7ed0e40a38e66b0e908560c5d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/a7bf3cdb216d4b65ac8145d8763de53c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/ec9910abc0ad4e6a8e6fd3a8e1d19be7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/8010395cf75e4712bff97c68765af222, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/6425720c792f4d299432ae88ab3e78db, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/719c1742141e4ad99d6264b659bb5f77, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/7f950f6bac704307a8d9d5f269bd2a3e] to archive 2024-11-17T01:52:42,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T01:52:42,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/ec69216d44034c89a85dee7fd3daf631 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/ec69216d44034c89a85dee7fd3daf631 2024-11-17T01:52:42,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/1392b9f817c74415944182303d9c06e0 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/1392b9f817c74415944182303d9c06e0 2024-11-17T01:52:42,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/7cd0262b733d4a84b4c35bf31f4e3b58 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/7cd0262b733d4a84b4c35bf31f4e3b58 2024-11-17T01:52:42,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/b08b643bec984b3e8cf1c09d04fbe3dd to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/b08b643bec984b3e8cf1c09d04fbe3dd 2024-11-17T01:52:42,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/0722c4e70eb048ae8223da516af80a63 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/0722c4e70eb048ae8223da516af80a63 2024-11-17T01:52:42,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/26e4ca2dd1b5442eb50eb608f8c1315e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/26e4ca2dd1b5442eb50eb608f8c1315e 2024-11-17T01:52:42,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/f22a4e6372574d10bd154bc4fe2b26a9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/f22a4e6372574d10bd154bc4fe2b26a9 2024-11-17T01:52:42,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/f67b15549d184fa5ae9503be14c41eb2 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/f67b15549d184fa5ae9503be14c41eb2 2024-11-17T01:52:42,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/b43903a6a40c4fa18ab46248f131b450 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/b43903a6a40c4fa18ab46248f131b450 2024-11-17T01:52:42,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/6ae8c476c110496a888b2cba9d60b0e2 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/6ae8c476c110496a888b2cba9d60b0e2 2024-11-17T01:52:42,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/fa239086aab547b2ad48d8f9ab9936c4 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/fa239086aab547b2ad48d8f9ab9936c4 2024-11-17T01:52:42,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/4393d9ef630d402a8b475a2212912ce3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/4393d9ef630d402a8b475a2212912ce3 2024-11-17T01:52:42,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/6a4c9520593342c5b19d79226c9fd6f8 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/6a4c9520593342c5b19d79226c9fd6f8 2024-11-17T01:52:42,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/a1705ce3a47342228f821ffffa0e11e1 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/a1705ce3a47342228f821ffffa0e11e1 2024-11-17T01:52:42,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/28c79e82cf75476fa02a991ee2ce4746 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/28c79e82cf75476fa02a991ee2ce4746 2024-11-17T01:52:42,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/0ceddfa7ed0e40a38e66b0e908560c5d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/0ceddfa7ed0e40a38e66b0e908560c5d 2024-11-17T01:52:42,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/a7bf3cdb216d4b65ac8145d8763de53c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/a7bf3cdb216d4b65ac8145d8763de53c 2024-11-17T01:52:42,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/ec9910abc0ad4e6a8e6fd3a8e1d19be7 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/ec9910abc0ad4e6a8e6fd3a8e1d19be7 2024-11-17T01:52:42,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/8010395cf75e4712bff97c68765af222 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/8010395cf75e4712bff97c68765af222 2024-11-17T01:52:42,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/6425720c792f4d299432ae88ab3e78db to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/6425720c792f4d299432ae88ab3e78db 2024-11-17T01:52:42,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/719c1742141e4ad99d6264b659bb5f77 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/719c1742141e4ad99d6264b659bb5f77 2024-11-17T01:52:42,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/7f950f6bac704307a8d9d5f269bd2a3e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/7f950f6bac704307a8d9d5f269bd2a3e 2024-11-17T01:52:42,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a5f1c99026bc42ad846184b15ffd22ca, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/365835a01bfd4a76863db4753a179f4d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/333b18e121cc4616b391ff8b5a0dc9e5, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/18afd26658964ea19916b5d639b4afbf, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/98cc205c84ba46dba053207a1e91e347, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a8162ca9f0394d69af087e0f475ba54f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/b2891bd54f454cd1ae10872f2bcd6439, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/fbf1032a139746cbaf53184c7b672cd2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/bc16cefd15094dcf9511a5e67954c23f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a89e41895f4240bbbed4cab978cd7ad7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/6522972687fb47eab5e36e5c3a0a7334, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/3767cfc658c94b49a49d499cbb512ad3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/0bbb3e5d07df482198297e799dff0ebf, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/9f33597a6a5f486cb4e4c5a935126c9e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/25a64f771f9942bd92393c01f2a8414d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/3007b7ae55a64bfc9fc180235abf0a45, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/5cf84cc104a9463c9ed10c0facf6b532, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/874f188651ec4b418a33f004c8090d35, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/4d197e9e1b1440a28aaf560cface9f42, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/7b886d9af67444ecb1d6ee052e1f60b2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/f95f94a4065a4036960660e8d62966b2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/4b8026cb25f4474a8a2a817a8846cf94] to archive 2024-11-17T01:52:42,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T01:52:42,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a5f1c99026bc42ad846184b15ffd22ca to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a5f1c99026bc42ad846184b15ffd22ca 2024-11-17T01:52:42,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/365835a01bfd4a76863db4753a179f4d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/365835a01bfd4a76863db4753a179f4d 2024-11-17T01:52:42,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/333b18e121cc4616b391ff8b5a0dc9e5 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/333b18e121cc4616b391ff8b5a0dc9e5 2024-11-17T01:52:42,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/18afd26658964ea19916b5d639b4afbf to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/18afd26658964ea19916b5d639b4afbf 2024-11-17T01:52:42,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/98cc205c84ba46dba053207a1e91e347 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/98cc205c84ba46dba053207a1e91e347 2024-11-17T01:52:42,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a8162ca9f0394d69af087e0f475ba54f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a8162ca9f0394d69af087e0f475ba54f 2024-11-17T01:52:42,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/b2891bd54f454cd1ae10872f2bcd6439 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/b2891bd54f454cd1ae10872f2bcd6439 2024-11-17T01:52:42,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/fbf1032a139746cbaf53184c7b672cd2 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/fbf1032a139746cbaf53184c7b672cd2 2024-11-17T01:52:42,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/bc16cefd15094dcf9511a5e67954c23f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/bc16cefd15094dcf9511a5e67954c23f 2024-11-17T01:52:42,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a89e41895f4240bbbed4cab978cd7ad7 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a89e41895f4240bbbed4cab978cd7ad7 2024-11-17T01:52:42,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/6522972687fb47eab5e36e5c3a0a7334 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/6522972687fb47eab5e36e5c3a0a7334 2024-11-17T01:52:42,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/3767cfc658c94b49a49d499cbb512ad3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/3767cfc658c94b49a49d499cbb512ad3 2024-11-17T01:52:42,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/0bbb3e5d07df482198297e799dff0ebf to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/0bbb3e5d07df482198297e799dff0ebf 2024-11-17T01:52:42,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/9f33597a6a5f486cb4e4c5a935126c9e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/9f33597a6a5f486cb4e4c5a935126c9e 2024-11-17T01:52:42,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/25a64f771f9942bd92393c01f2a8414d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/25a64f771f9942bd92393c01f2a8414d 2024-11-17T01:52:42,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/3007b7ae55a64bfc9fc180235abf0a45 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/3007b7ae55a64bfc9fc180235abf0a45 2024-11-17T01:52:42,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/5cf84cc104a9463c9ed10c0facf6b532 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/5cf84cc104a9463c9ed10c0facf6b532 2024-11-17T01:52:42,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/874f188651ec4b418a33f004c8090d35 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/874f188651ec4b418a33f004c8090d35 2024-11-17T01:52:42,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/4d197e9e1b1440a28aaf560cface9f42 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/4d197e9e1b1440a28aaf560cface9f42 2024-11-17T01:52:42,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/7b886d9af67444ecb1d6ee052e1f60b2 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/7b886d9af67444ecb1d6ee052e1f60b2 2024-11-17T01:52:42,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/f95f94a4065a4036960660e8d62966b2 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/f95f94a4065a4036960660e8d62966b2 2024-11-17T01:52:42,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/4b8026cb25f4474a8a2a817a8846cf94 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/4b8026cb25f4474a8a2a817a8846cf94 2024-11-17T01:52:42,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/ef894734631c406a90fbf7555505c101, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/dad19ea2b9df438882675d8cb018f0b6, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/ff45aaceb6fe4384a0b562045b199b39, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/4ca0f15d92ff4c3db49a229a01d63dc0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/73ecf5ab05494337ac3e28e689b7999a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/daa3622ac2b34adcba975422a05c8b19, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/76f2a413fe9b468db1ebee83d1aa6321, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/ce1396d0aff7419e8db93c4d6b9309fc, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/499e06c1f40741599d528a46f34d1d0a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/e5446c498e8c41e99f94cc48e50103ba, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/c91beacd3c7248b784767e926cccb187, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/41cd3899f765425b92a8c73d2ed5d4c7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/298fffbf7de647038428438953c01f20, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/4e34bf769939445080614decda4f761c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/b01c9b4a64824cd2869de4a9bbc64d4f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/4b8206b92899475ea92b2ae28956c429, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/0ae1333aac544e02830e65c92f979428, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/51b6a3ab67504c9387e4c9847771f2a9, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/5a066c6f08cf43c9b7634694b3136dfd, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/a028e2a0082d42b5aa5f4a84db2dda2f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/d631f84f5b1a4ae1bde6ba19bfa0ccef, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/6614810f187b497181d815fcc761b812] to archive 2024-11-17T01:52:42,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T01:52:42,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/ef894734631c406a90fbf7555505c101 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/ef894734631c406a90fbf7555505c101 2024-11-17T01:52:42,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/dad19ea2b9df438882675d8cb018f0b6 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/dad19ea2b9df438882675d8cb018f0b6 2024-11-17T01:52:42,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/ff45aaceb6fe4384a0b562045b199b39 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/ff45aaceb6fe4384a0b562045b199b39 2024-11-17T01:52:42,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/4ca0f15d92ff4c3db49a229a01d63dc0 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/4ca0f15d92ff4c3db49a229a01d63dc0 2024-11-17T01:52:42,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/73ecf5ab05494337ac3e28e689b7999a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/73ecf5ab05494337ac3e28e689b7999a 2024-11-17T01:52:42,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/daa3622ac2b34adcba975422a05c8b19 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/daa3622ac2b34adcba975422a05c8b19 2024-11-17T01:52:42,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/76f2a413fe9b468db1ebee83d1aa6321 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/76f2a413fe9b468db1ebee83d1aa6321 2024-11-17T01:52:42,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/ce1396d0aff7419e8db93c4d6b9309fc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/ce1396d0aff7419e8db93c4d6b9309fc 2024-11-17T01:52:42,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/499e06c1f40741599d528a46f34d1d0a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/499e06c1f40741599d528a46f34d1d0a 2024-11-17T01:52:42,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/e5446c498e8c41e99f94cc48e50103ba to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/e5446c498e8c41e99f94cc48e50103ba 2024-11-17T01:52:42,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/c91beacd3c7248b784767e926cccb187 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/c91beacd3c7248b784767e926cccb187 2024-11-17T01:52:42,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/41cd3899f765425b92a8c73d2ed5d4c7 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/41cd3899f765425b92a8c73d2ed5d4c7 2024-11-17T01:52:42,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/298fffbf7de647038428438953c01f20 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/298fffbf7de647038428438953c01f20 2024-11-17T01:52:42,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/4e34bf769939445080614decda4f761c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/4e34bf769939445080614decda4f761c 2024-11-17T01:52:42,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/b01c9b4a64824cd2869de4a9bbc64d4f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/b01c9b4a64824cd2869de4a9bbc64d4f 2024-11-17T01:52:42,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/4b8206b92899475ea92b2ae28956c429 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/4b8206b92899475ea92b2ae28956c429 2024-11-17T01:52:42,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/0ae1333aac544e02830e65c92f979428 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/0ae1333aac544e02830e65c92f979428 2024-11-17T01:52:42,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/51b6a3ab67504c9387e4c9847771f2a9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/51b6a3ab67504c9387e4c9847771f2a9 2024-11-17T01:52:42,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/5a066c6f08cf43c9b7634694b3136dfd to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/5a066c6f08cf43c9b7634694b3136dfd 2024-11-17T01:52:42,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/a028e2a0082d42b5aa5f4a84db2dda2f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/a028e2a0082d42b5aa5f4a84db2dda2f 2024-11-17T01:52:42,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/d631f84f5b1a4ae1bde6ba19bfa0ccef to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/d631f84f5b1a4ae1bde6ba19bfa0ccef 2024-11-17T01:52:42,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/6614810f187b497181d815fcc761b812 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/6614810f187b497181d815fcc761b812 2024-11-17T01:52:42,776 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/recovered.edits/313.seqid, newMaxSeqId=313, maxSeqId=4 2024-11-17T01:52:42,777 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. 2024-11-17T01:52:42,777 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 619e0c9a5ad3ede5c5b6d8d44658cb1a: Waiting for close lock at 1731808362172Running coprocessor pre-close hooks at 1731808362172Disabling compacts and flushes for region at 1731808362172Disabling writes for close at 1731808362172Obtaining lock to block concurrent updates at 1731808362172Preparing flush snapshotting stores in 619e0c9a5ad3ede5c5b6d8d44658cb1a at 1731808362172Finished memstore snapshotting TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731808362173 (+1 ms)Flushing stores of TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a. at 1731808362174 (+1 ms)Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a/A: creating writer at 1731808362174Flushing mob file 619e0c9a5ad3ede5c5b6d8d44658cb1a/A: appending metadata at 1731808362180 (+6 ms)Flushing mob file 619e0c9a5ad3ede5c5b6d8d44658cb1a/A: closing flushed file at 1731808362180Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a/A: appending metadata at 1731808362196 (+16 ms)Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a/A: closing flushed file at 1731808362196Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a/B: creating writer at 1731808362213 (+17 ms)Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a/B: appending metadata at 1731808362216 (+3 ms)Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a/B: closing flushed file at 1731808362216Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a/C: creating writer at 1731808362228 (+12 ms)Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a/C: appending metadata at 1731808362231 (+3 ms)Flushing 619e0c9a5ad3ede5c5b6d8d44658cb1a/C: closing flushed file at 1731808362231Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10cc5582: reopening flushed file at 1731808362642 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f54e641: reopening flushed file at 1731808362648 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d949440: reopening flushed file at 1731808362654 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 619e0c9a5ad3ede5c5b6d8d44658cb1a in 489ms, sequenceid=310, compaction requested=false at 1731808362661 (+7 ms)Writing region close event to WAL at 1731808362773 (+112 ms)Running coprocessor post-close hooks at 1731808362777 (+4 ms)Closed at 1731808362777 2024-11-17T01:52:42,780 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:42,781 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=619e0c9a5ad3ede5c5b6d8d44658cb1a, regionState=CLOSED 2024-11-17T01:52:42,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 because future has completed 2024-11-17T01:52:42,787 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-17T01:52:42,787 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 619e0c9a5ad3ede5c5b6d8d44658cb1a, server=c6c0d7a0a7c0,41157,1731808292173 in 765 msec 2024-11-17T01:52:42,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-17T01:52:42,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619e0c9a5ad3ede5c5b6d8d44658cb1a, UNASSIGN in 772 msec 2024-11-17T01:52:42,792 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-17T01:52:42,792 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 777 msec 2024-11-17T01:52:42,794 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808362794"}]},"ts":"1731808362794"} 2024-11-17T01:52:42,796 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T01:52:42,796 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T01:52:42,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 789 msec 2024-11-17T01:52:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T01:52:43,144 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:43,145 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T01:52:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:52:43,147 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:52:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T01:52:43,148 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:52:43,153 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:43,155 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A, FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B, FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C, FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/recovered.edits] 2024-11-17T01:52:43,158 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/057210b8586f4a1da07b2ff1a86685d7 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/057210b8586f4a1da07b2ff1a86685d7 2024-11-17T01:52:43,160 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/7483419f63914f62ba812e1b3a25a3f3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/A/7483419f63914f62ba812e1b3a25a3f3 2024-11-17T01:52:43,163 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/8c61b72d933843b59e0f8b7377f52539 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/8c61b72d933843b59e0f8b7377f52539 2024-11-17T01:52:43,164 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a23ef7e3e62e4359ad5af46c4abc867e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/B/a23ef7e3e62e4359ad5af46c4abc867e 2024-11-17T01:52:43,167 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/96a3ab5af25a41208d1cda3971859414 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/96a3ab5af25a41208d1cda3971859414 2024-11-17T01:52:43,169 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/fb8456418fb142fa868f2f026cddceb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/C/fb8456418fb142fa868f2f026cddceb9 2024-11-17T01:52:43,172 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/recovered.edits/313.seqid to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a/recovered.edits/313.seqid 2024-11-17T01:52:43,173 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:43,173 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T01:52:43,174 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T01:52:43,175 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T01:52:43,181 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111704437e0471d04fa990e6c00d42bd9a76_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111704437e0471d04fa990e6c00d42bd9a76_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:43,182 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111705ac2e2fd7a14687af80b1d1e78d7f24_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111705ac2e2fd7a14687af80b1d1e78d7f24_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:43,184 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170ec0bfe33cd243cd934705489112ec9e_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170ec0bfe33cd243cd934705489112ec9e_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:43,185 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111730fa010ccbd54dab8c2817d063515456_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111730fa010ccbd54dab8c2817d063515456_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:43,187 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117356fc0d3adc2472d9ea9a0ad7e2b60ee_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117356fc0d3adc2472d9ea9a0ad7e2b60ee_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:43,188 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117395281ccd19a4ade85b0c4eefd2a72fb_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117395281ccd19a4ade85b0c4eefd2a72fb_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:43,191 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111744393503042e42f7af39630afc020273_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111744393503042e42f7af39630afc020273_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:43,193 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174470583b1b2d4454b2a71fdc8c6197ad_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174470583b1b2d4454b2a71fdc8c6197ad_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:43,194 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175aa8a4a6e90b4949bf318ab029e4da76_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175aa8a4a6e90b4949bf318ab029e4da76_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:43,196 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111767a65353715542ce89e0db85ee322fd9_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111767a65353715542ce89e0db85ee322fd9_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:43,197 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177d4e8e038e944628a722a11ca4fe81ae_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177d4e8e038e944628a722a11ca4fe81ae_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:43,199 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178099c2b9147f476eb4fad723de67db86_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178099c2b9147f476eb4fad723de67db86_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:43,201 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a706e35a5339481fabfd0b1b4d3d6c68_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a706e35a5339481fabfd0b1b4d3d6c68_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:43,203 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b3ab4063ba1742ffa787f6dd0784252b_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b3ab4063ba1742ffa787f6dd0784252b_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:43,205 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d59bb25d5354419c85d6654e9ac59743_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d59bb25d5354419c85d6654e9ac59743_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:43,207 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117df2e04c144f34bff930da8e9b8506b75_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117df2e04c144f34bff930da8e9b8506b75_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:43,209 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f32263ee3f7f490cba7ba9ba54615b6c_619e0c9a5ad3ede5c5b6d8d44658cb1a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f32263ee3f7f490cba7ba9ba54615b6c_619e0c9a5ad3ede5c5b6d8d44658cb1a 2024-11-17T01:52:43,210 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T01:52:43,215 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:52:43,219 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T01:52:43,222 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T01:52:43,223 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:52:43,224 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T01:52:43,224 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731808363224"}]},"ts":"9223372036854775807"} 2024-11-17T01:52:43,227 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T01:52:43,227 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 619e0c9a5ad3ede5c5b6d8d44658cb1a, NAME => 'TestAcidGuarantees,,1731808330309.619e0c9a5ad3ede5c5b6d8d44658cb1a.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T01:52:43,228 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T01:52:43,228 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731808363228"}]},"ts":"9223372036854775807"} 2024-11-17T01:52:43,230 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T01:52:43,231 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:52:43,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 86 msec 2024-11-17T01:52:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T01:52:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T01:52:43,255 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:43,266 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 247) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1864014360_22 at /127.0.0.1:46556 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/cluster_97028d77-d23a-8219-e606-b0c441f49b52/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/cluster_97028d77-d23a-8219-e606-b0c441f49b52/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=453 (was 345) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5022 (was 5093) 2024-11-17T01:52:43,277 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=453, ProcessCount=11, AvailableMemoryMB=5022 2024-11-17T01:52:43,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T01:52:43,279 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:52:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T01:52:43,282 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T01:52:43,282 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:43,282 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-17T01:52:43,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-17T01:52:43,283 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T01:52:43,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742047_1223 (size=1061) 2024-11-17T01:52:43,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-17T01:52:43,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-17T01:52:43,692 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1 2024-11-17T01:52:43,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742048_1224 (size=53) 2024-11-17T01:52:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-17T01:52:44,138 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:52:44,138 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 36d65c1d5dfe82a6655fada53119d841, disabling compactions & flushes 2024-11-17T01:52:44,138 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:44,138 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:44,138 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. after waiting 0 ms 2024-11-17T01:52:44,139 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:44,139 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:44,139 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 36d65c1d5dfe82a6655fada53119d841: Waiting for close lock at 1731808364138Disabling compacts and flushes for region at 1731808364138Disabling writes for close at 1731808364138Writing region close event to WAL at 1731808364139 (+1 ms)Closed at 1731808364139 2024-11-17T01:52:44,140 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T01:52:44,140 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731808364140"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731808364140"}]},"ts":"1731808364140"} 2024-11-17T01:52:44,142 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T01:52:44,146 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T01:52:44,147 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808364146"}]},"ts":"1731808364146"} 2024-11-17T01:52:44,149 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T01:52:44,150 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=36d65c1d5dfe82a6655fada53119d841, ASSIGN}] 2024-11-17T01:52:44,151 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=36d65c1d5dfe82a6655fada53119d841, ASSIGN 2024-11-17T01:52:44,152 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=36d65c1d5dfe82a6655fada53119d841, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,41157,1731808292173; forceNewPlan=false, retain=false 2024-11-17T01:52:44,303 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=36d65c1d5dfe82a6655fada53119d841, regionState=OPENING, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:44,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=36d65c1d5dfe82a6655fada53119d841, ASSIGN because future has completed 2024-11-17T01:52:44,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173}] 2024-11-17T01:52:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-17T01:52:44,463 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:44,464 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:52:44,464 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:44,465 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:52:44,465 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:44,465 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:44,467 INFO [StoreOpener-36d65c1d5dfe82a6655fada53119d841-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:44,468 INFO [StoreOpener-36d65c1d5dfe82a6655fada53119d841-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:52:44,468 INFO [StoreOpener-36d65c1d5dfe82a6655fada53119d841-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36d65c1d5dfe82a6655fada53119d841 columnFamilyName A 2024-11-17T01:52:44,468 DEBUG [StoreOpener-36d65c1d5dfe82a6655fada53119d841-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:44,469 INFO [StoreOpener-36d65c1d5dfe82a6655fada53119d841-1 {}] regionserver.HStore(327): Store=36d65c1d5dfe82a6655fada53119d841/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:52:44,469 INFO [StoreOpener-36d65c1d5dfe82a6655fada53119d841-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:44,470 INFO [StoreOpener-36d65c1d5dfe82a6655fada53119d841-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:52:44,471 INFO [StoreOpener-36d65c1d5dfe82a6655fada53119d841-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36d65c1d5dfe82a6655fada53119d841 columnFamilyName B 2024-11-17T01:52:44,471 DEBUG [StoreOpener-36d65c1d5dfe82a6655fada53119d841-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:44,471 INFO [StoreOpener-36d65c1d5dfe82a6655fada53119d841-1 {}] regionserver.HStore(327): Store=36d65c1d5dfe82a6655fada53119d841/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:52:44,471 INFO [StoreOpener-36d65c1d5dfe82a6655fada53119d841-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:44,473 INFO [StoreOpener-36d65c1d5dfe82a6655fada53119d841-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:52:44,473 INFO [StoreOpener-36d65c1d5dfe82a6655fada53119d841-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36d65c1d5dfe82a6655fada53119d841 columnFamilyName C 2024-11-17T01:52:44,473 DEBUG [StoreOpener-36d65c1d5dfe82a6655fada53119d841-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:52:44,473 INFO [StoreOpener-36d65c1d5dfe82a6655fada53119d841-1 {}] regionserver.HStore(327): Store=36d65c1d5dfe82a6655fada53119d841/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:52:44,474 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:44,474 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:44,474 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:44,475 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:44,476 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:44,476 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:44,477 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T01:52:44,478 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:44,480 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:52:44,481 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 36d65c1d5dfe82a6655fada53119d841; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66643597, jitterRate=-0.0069330185651779175}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T01:52:44,481 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:44,481 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 36d65c1d5dfe82a6655fada53119d841: Running coprocessor pre-open hook at 1731808364465Writing region info on filesystem at 1731808364465Initializing all the Stores at 1731808364466 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808364466Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808364466Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808364467 (+1 ms)Cleaning up temporary data from old regions at 1731808364476 (+9 ms)Running coprocessor post-open hooks at 1731808364481 (+5 ms)Region opened successfully at 1731808364481 2024-11-17T01:52:44,482 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., pid=61, masterSystemTime=1731808364457 2024-11-17T01:52:44,484 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:44,484 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:44,484 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=36d65c1d5dfe82a6655fada53119d841, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:44,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 because future has completed 2024-11-17T01:52:44,489 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-17T01:52:44,489 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 in 183 msec 2024-11-17T01:52:44,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-17T01:52:44,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=36d65c1d5dfe82a6655fada53119d841, ASSIGN in 339 msec 2024-11-17T01:52:44,492 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T01:52:44,492 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808364492"}]},"ts":"1731808364492"} 2024-11-17T01:52:44,494 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T01:52:44,495 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T01:52:44,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2160 sec 2024-11-17T01:52:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-17T01:52:45,434 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:45,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,435 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:45,435 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:45,436 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:45,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a9e3f98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:45,438 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185c00db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,439 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:45,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:45,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:45,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:45,441 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T01:52:45,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is c6c0d7a0a7c0,45023,1731808291372 2024-11-17T01:52:45,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@218c79d0 2024-11-17T01:52:45,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T01:52:45,443 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T01:52:45,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,444 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:45,444 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:45,445 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1616023b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:45,446 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e001613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,447 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:45,447 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:45,448 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ef5e5f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:45,449 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:45,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d2f143, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,450 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:45,450 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:45,451 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:45,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:45,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:45,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31cf1bfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:45,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:45,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:45,452 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd79328, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,453 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:45,453 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:45,453 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27174327, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:45,454 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:45,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7398af7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,455 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:45,456 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:45,456 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:45,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:45,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705da212, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:45,457 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@454489, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,458 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,458 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:45,459 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:45,459 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f8d26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:45,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:45,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:45,460 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77849c35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,461 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,461 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:45,461 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:45,462 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@708412a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:45,463 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:45,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528ae5de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,464 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:45,464 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:45,464 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7277a430, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:45,465 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:45,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33527d26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,466 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:45,466 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:45,467 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@703df9ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:45,468 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f83647d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,469 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,469 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:52:45,469 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:52:45,470 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:52:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:52:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:52:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d6c0fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:52:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:52:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:52:45,471 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:52:45,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41158dea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:52:45,472 DEBUG [Thread-1058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:45,472 DEBUG [Thread-1073 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:45,472 DEBUG [Thread-1062 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:45,473 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:52:45,473 DEBUG [Thread-1067 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:45,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:45,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:45,473 DEBUG [Thread-1071 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:45,473 DEBUG [Thread-1065 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:45,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:45,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:45,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:45,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:45,474 DEBUG [Thread-1065 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-17T01:52:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:45,474 DEBUG [Thread-1065 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:45,474 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:45,474 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:45,475 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:52:45,475 DEBUG [Thread-1054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T01:52:45,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:45,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:45,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:45,476 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:52:45,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:45,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:52:45,476 DEBUG [Thread-1069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:45,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:45,476 DEBUG [Thread-1056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:45,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:45,477 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:45,477 DEBUG [Thread-1069 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:45,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:45,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:52:45,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:45,478 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:45,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:52:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:52:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:52:45,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:52:45,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:52:45,480 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:45,480 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:45,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:52:45,481 DEBUG [Thread-1060 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:52:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:52:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:52:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:52:45,483 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:52:45,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:52:45,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:52:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:45,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T01:52:45,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:52:45,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:45,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:52:45,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:45,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:52:45,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:45,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/312e0d64b11e44b1b4901fe435eafcee is 50, key is test_row_0/A:col10/1731808365487/Put/seqid=0 2024-11-17T01:52:45,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808425524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:45,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808425524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:45,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808365508, max timestamp=1731808365531Num uniques 859; 2024-11-17T01:52:45,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808365508, max timestamp=1731808365531Num uniques 859; 2024-11-17T01:52:45,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:52:45,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:52:45,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808365508, max timestamp=1731808365531Num uniques 859; 2024-11-17T01:52:45,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:52:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:45,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:45,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742049_1225 (size=12001) 2024-11-17T01:52:45,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/312e0d64b11e44b1b4901fe435eafcee 2024-11-17T01:52:45,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/20dc0285c3b7433bba05c4cd095ff9b0 is 50, key is test_row_0/B:col10/1731808365487/Put/seqid=0 2024-11-17T01:52:45,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44270 deadline: 1731808425574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:45,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44298 deadline: 1731808425576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:45,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T01:52:45,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44294 deadline: 1731808425580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:45,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:45,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742050_1226 (size=12001) 2024-11-17T01:52:45,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/20dc0285c3b7433bba05c4cd095ff9b0 2024-11-17T01:52:45,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T01:52:45,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:45,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:45,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:45,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:45,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:45,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808425644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:45,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:45,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808425644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:45,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/377a8b050eae47ee9bd1aef069d7232b is 50, key is test_row_0/C:col10/1731808365487/Put/seqid=0 2024-11-17T01:52:45,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44270 deadline: 1731808425684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:45,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44298 deadline: 1731808425684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:45,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:45,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:45,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742051_1227 (size=12001) 2024-11-17T01:52:45,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/377a8b050eae47ee9bd1aef069d7232b 2024-11-17T01:52:45,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44294 deadline: 1731808425694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:45,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:45,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/312e0d64b11e44b1b4901fe435eafcee as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/312e0d64b11e44b1b4901fe435eafcee 2024-11-17T01:52:45,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/312e0d64b11e44b1b4901fe435eafcee, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T01:52:45,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/20dc0285c3b7433bba05c4cd095ff9b0 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/20dc0285c3b7433bba05c4cd095ff9b0 2024-11-17T01:52:45,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/20dc0285c3b7433bba05c4cd095ff9b0, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T01:52:45,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/377a8b050eae47ee9bd1aef069d7232b as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/377a8b050eae47ee9bd1aef069d7232b 2024-11-17T01:52:45,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/377a8b050eae47ee9bd1aef069d7232b, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T01:52:45,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 36d65c1d5dfe82a6655fada53119d841 in 239ms, sequenceid=12, compaction requested=false 2024-11-17T01:52:45,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:45,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T01:52:45,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:45,788 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-17T01:52:45,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:52:45,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:52:45,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:52:45,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:52:45,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:52:45,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:52:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T01:52:45,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/736f4aae2b0245da9363906198b7f2d5 is 50, key is test_row_0/A:col10/1731808365568/Put/seqid=0 2024-11-17T01:52:45,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742052_1228 (size=12001) 2024-11-17T01:52:45,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:45,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:45,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808425874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:45,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:45,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808425874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:45,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:45,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:45,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44270 deadline: 1731808425894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:45,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:45,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44298 deadline: 1731808425894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:45,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:45,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:45,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44294 deadline: 1731808425904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:45,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:45,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808425984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:45,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808425984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T01:52:46,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808426194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:46,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:46,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:46,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808426194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:46,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:46,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:46,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:46,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:46,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44270 deadline: 1731808426204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:46,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44298 deadline: 1731808426204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:46,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:46,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:46,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:46,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:46,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:46,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:46,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44294 deadline: 1731808426213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:46,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:46,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:46,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:46,243 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/736f4aae2b0245da9363906198b7f2d5 2024-11-17T01:52:46,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/47c3b45deb29496693a78d89ca9b1f39 is 50, key is test_row_0/B:col10/1731808365568/Put/seqid=0 2024-11-17T01:52:46,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742053_1229 (size=12001) 2024-11-17T01:52:46,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:46,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808426504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:46,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:46,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:46,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:46,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:46,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808426504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:46,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:46,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:46,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T01:52:46,659 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/47c3b45deb29496693a78d89ca9b1f39 2024-11-17T01:52:46,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/99338179099442b2bb829bab34d86d06 is 50, key is test_row_0/C:col10/1731808365568/Put/seqid=0 2024-11-17T01:52:46,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742054_1230 (size=12001) 2024-11-17T01:52:46,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44298 deadline: 1731808426714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:46,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44270 deadline: 1731808426714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:46,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44294 deadline: 1731808426724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:46,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:46,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:46,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:47,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808427014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:47,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:47,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808427014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:47,070 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/99338179099442b2bb829bab34d86d06 2024-11-17T01:52:47,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/736f4aae2b0245da9363906198b7f2d5 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/736f4aae2b0245da9363906198b7f2d5 2024-11-17T01:52:47,079 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/736f4aae2b0245da9363906198b7f2d5, entries=150, sequenceid=47, filesize=11.7 K 2024-11-17T01:52:47,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/47c3b45deb29496693a78d89ca9b1f39 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/47c3b45deb29496693a78d89ca9b1f39 2024-11-17T01:52:47,084 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/47c3b45deb29496693a78d89ca9b1f39, entries=150, sequenceid=47, filesize=11.7 K 2024-11-17T01:52:47,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/99338179099442b2bb829bab34d86d06 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/99338179099442b2bb829bab34d86d06 2024-11-17T01:52:47,089 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/99338179099442b2bb829bab34d86d06, entries=150, sequenceid=47, filesize=11.7 K 2024-11-17T01:52:47,089 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 36d65c1d5dfe82a6655fada53119d841 in 1301ms, sequenceid=47, compaction requested=false 2024-11-17T01:52:47,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:47,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:47,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-17T01:52:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-17T01:52:47,092 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-17T01:52:47,092 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6150 sec 2024-11-17T01:52:47,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.6200 sec 2024-11-17T01:52:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T01:52:47,624 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:47,625 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:52:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-17T01:52:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T01:52:47,627 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:52:47,627 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:52:47,627 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:52:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T01:52:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:47,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T01:52:47,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:52:47,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:47,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:52:47,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:47,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:52:47,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:47,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/71f59d1044684ca4b30a57277dcaecc1 is 50, key is test_row_0/A:col10/1731808365872/Put/seqid=0 2024-11-17T01:52:47,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742055_1231 (size=12001) 2024-11-17T01:52:47,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/71f59d1044684ca4b30a57277dcaecc1 2024-11-17T01:52:47,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/b13e01c4d56c44528f1506c265f04e58 is 50, key is test_row_0/B:col10/1731808365872/Put/seqid=0 2024-11-17T01:52:47,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:47,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44294 deadline: 1731808427769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44298 deadline: 1731808427769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:47,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44270 deadline: 1731808427774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:47,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T01:52:47,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:47,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:47,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:47,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:47,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:47,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742056_1232 (size=12001) 2024-11-17T01:52:47,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44294 deadline: 1731808427884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:47,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:47,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:47,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:47,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44298 deadline: 1731808427884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:47,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:47,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:47,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44270 deadline: 1731808427884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:47,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T01:52:47,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:47,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:47,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:47,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:47,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T01:52:48,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808428034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:48,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808428034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:48,038 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in 36d65c1d5dfe82a6655fada53119d841 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:52:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,038 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in 36d65c1d5dfe82a6655fada53119d841 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:52:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:48,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T01:52:48,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:48,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:48,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:48,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:48,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:48,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:48,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44294 deadline: 1731808428094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:48,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44298 deadline: 1731808428094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44270 deadline: 1731808428094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:48,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/b13e01c4d56c44528f1506c265f04e58 2024-11-17T01:52:48,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/1a5ebf8fcf484f8d9d79df5c72af16f6 is 50, key is test_row_0/C:col10/1731808365872/Put/seqid=0 2024-11-17T01:52:48,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742057_1233 (size=12001) 2024-11-17T01:52:48,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T01:52:48,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:48,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:48,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:48,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:48,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T01:52:48,305 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T01:52:48,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T01:52:48,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:48,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:48,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:48,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:48,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:48,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44294 deadline: 1731808428404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:48,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44298 deadline: 1731808428404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:48,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44270 deadline: 1731808428404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:48,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T01:52:48,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:48,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:48,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:48,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:48,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:48,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/1a5ebf8fcf484f8d9d79df5c72af16f6 2024-11-17T01:52:48,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/71f59d1044684ca4b30a57277dcaecc1 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/71f59d1044684ca4b30a57277dcaecc1 2024-11-17T01:52:48,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/71f59d1044684ca4b30a57277dcaecc1, entries=150, sequenceid=60, filesize=11.7 K 2024-11-17T01:52:48,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/b13e01c4d56c44528f1506c265f04e58 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/b13e01c4d56c44528f1506c265f04e58 2024-11-17T01:52:48,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/b13e01c4d56c44528f1506c265f04e58, entries=150, sequenceid=60, filesize=11.7 K 2024-11-17T01:52:48,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/1a5ebf8fcf484f8d9d79df5c72af16f6 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/1a5ebf8fcf484f8d9d79df5c72af16f6 2024-11-17T01:52:48,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/1a5ebf8fcf484f8d9d79df5c72af16f6, entries=150, sequenceid=60, filesize=11.7 K 2024-11-17T01:52:48,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 36d65c1d5dfe82a6655fada53119d841 in 923ms, sequenceid=60, compaction requested=true 2024-11-17T01:52:48,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:48,660 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:48,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:52:48,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:48,660 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:48,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:52:48,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:48,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:52:48,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:48,661 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:48,661 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/A is initiating minor compaction (all files) 2024-11-17T01:52:48,661 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:48,661 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/A in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:48,661 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/B is initiating minor compaction (all files) 2024-11-17T01:52:48,661 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/B in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:48,661 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/312e0d64b11e44b1b4901fe435eafcee, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/736f4aae2b0245da9363906198b7f2d5, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/71f59d1044684ca4b30a57277dcaecc1] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=35.2 K 2024-11-17T01:52:48,661 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/20dc0285c3b7433bba05c4cd095ff9b0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/47c3b45deb29496693a78d89ca9b1f39, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/b13e01c4d56c44528f1506c265f04e58] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=35.2 K 2024-11-17T01:52:48,662 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 312e0d64b11e44b1b4901fe435eafcee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731808365487 2024-11-17T01:52:48,662 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 20dc0285c3b7433bba05c4cd095ff9b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731808365487 2024-11-17T01:52:48,662 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 736f4aae2b0245da9363906198b7f2d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731808365568 2024-11-17T01:52:48,663 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71f59d1044684ca4b30a57277dcaecc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731808365872 2024-11-17T01:52:48,663 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 47c3b45deb29496693a78d89ca9b1f39, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731808365568 2024-11-17T01:52:48,663 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting b13e01c4d56c44528f1506c265f04e58, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731808365872 2024-11-17T01:52:48,684 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#A#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:48,685 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/0b34cf11592a47d89f2eb9793a96ab72 is 50, key is test_row_0/A:col10/1731808365872/Put/seqid=0 2024-11-17T01:52:48,701 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#B#compaction#193 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:48,702 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/63cd5dc383a341159d3f0e3b24784296 is 50, key is test_row_0/B:col10/1731808365872/Put/seqid=0 2024-11-17T01:52:48,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T01:52:48,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:48,702 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T01:52:48,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:52:48,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:48,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:52:48,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:48,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:52:48,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:48,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742058_1234 (size=12104) 2024-11-17T01:52:48,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/735702213e704c16ba6ea428958b3a5d is 50, key is test_row_0/A:col10/1731808367773/Put/seqid=0 2024-11-17T01:52:48,739 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/0b34cf11592a47d89f2eb9793a96ab72 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/0b34cf11592a47d89f2eb9793a96ab72 2024-11-17T01:52:48,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742059_1235 (size=12104) 2024-11-17T01:52:48,753 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/A of 36d65c1d5dfe82a6655fada53119d841 into 0b34cf11592a47d89f2eb9793a96ab72(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:48,753 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:48,753 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/A, priority=13, startTime=1731808368660; duration=0sec 2024-11-17T01:52:48,753 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:48,753 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:A 2024-11-17T01:52:48,753 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:48,759 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:48,759 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/C is initiating minor compaction (all files) 2024-11-17T01:52:48,759 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/C in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:48,759 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/377a8b050eae47ee9bd1aef069d7232b, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/99338179099442b2bb829bab34d86d06, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/1a5ebf8fcf484f8d9d79df5c72af16f6] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=35.2 K 2024-11-17T01:52:48,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742060_1236 (size=12001) 2024-11-17T01:52:48,760 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 377a8b050eae47ee9bd1aef069d7232b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731808365487 2024-11-17T01:52:48,761 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99338179099442b2bb829bab34d86d06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731808365568 2024-11-17T01:52:48,761 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/735702213e704c16ba6ea428958b3a5d 2024-11-17T01:52:48,761 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a5ebf8fcf484f8d9d79df5c72af16f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731808365872 2024-11-17T01:52:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T01:52:48,766 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/63cd5dc383a341159d3f0e3b24784296 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/63cd5dc383a341159d3f0e3b24784296 2024-11-17T01:52:48,776 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#C#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:48,776 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/445cc0befa8b471fb3a55b8b7ca1791c is 50, key is test_row_0/C:col10/1731808365872/Put/seqid=0 2024-11-17T01:52:48,781 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/B of 36d65c1d5dfe82a6655fada53119d841 into 63cd5dc383a341159d3f0e3b24784296(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:48,781 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:48,781 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/B, priority=13, startTime=1731808368660; duration=0sec 2024-11-17T01:52:48,781 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:48,781 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:B 2024-11-17T01:52:48,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/cbe9ec48709b4c6083a1d968cfdfa36d is 50, key is test_row_0/B:col10/1731808367773/Put/seqid=0 2024-11-17T01:52:48,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742061_1237 (size=12104) 2024-11-17T01:52:48,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742062_1238 (size=12001) 2024-11-17T01:52:48,852 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/cbe9ec48709b4c6083a1d968cfdfa36d 2024-11-17T01:52:48,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/9e86b203407545edbca37bc5b8903f78 is 50, key is test_row_0/C:col10/1731808367773/Put/seqid=0 2024-11-17T01:52:48,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742063_1239 (size=12001) 2024-11-17T01:52:48,911 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/9e86b203407545edbca37bc5b8903f78 2024-11-17T01:52:48,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:48,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/735702213e704c16ba6ea428958b3a5d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/735702213e704c16ba6ea428958b3a5d 2024-11-17T01:52:48,933 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/735702213e704c16ba6ea428958b3a5d, entries=150, sequenceid=79, filesize=11.7 K 2024-11-17T01:52:48,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/cbe9ec48709b4c6083a1d968cfdfa36d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/cbe9ec48709b4c6083a1d968cfdfa36d 2024-11-17T01:52:48,941 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/cbe9ec48709b4c6083a1d968cfdfa36d, entries=150, sequenceid=79, filesize=11.7 K 2024-11-17T01:52:48,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/9e86b203407545edbca37bc5b8903f78 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/9e86b203407545edbca37bc5b8903f78 2024-11-17T01:52:48,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44294 deadline: 1731808428943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:48,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:48,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44298 deadline: 1731808428943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:48,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44270 deadline: 1731808428945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:48,952 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/9e86b203407545edbca37bc5b8903f78, entries=150, sequenceid=79, filesize=11.7 K 2024-11-17T01:52:48,954 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 36d65c1d5dfe82a6655fada53119d841 in 251ms, sequenceid=79, compaction requested=false 2024-11-17T01:52:48,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:48,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:48,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-17T01:52:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-17T01:52:48,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-17T01:52:48,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3280 sec 2024-11-17T01:52:48,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.3330 sec 2024-11-17T01:52:49,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:49,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T01:52:49,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:52:49,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:49,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:52:49,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:49,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:52:49,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:49,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/11876b0ad9104e9a8de810af17707933 is 50, key is test_row_0/A:col10/1731808369057/Put/seqid=0 2024-11-17T01:52:49,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44298 deadline: 1731808429088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:49,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:49,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44270 deadline: 1731808429089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:49,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:49,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:49,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742064_1240 (size=14341) 2024-11-17T01:52:49,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:49,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44294 deadline: 1731808429096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:49,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:49,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:49,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44270 deadline: 1731808429204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:49,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44298 deadline: 1731808429204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:49,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:49,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44294 deadline: 1731808429214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:49,242 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/445cc0befa8b471fb3a55b8b7ca1791c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/445cc0befa8b471fb3a55b8b7ca1791c 2024-11-17T01:52:49,248 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/C of 36d65c1d5dfe82a6655fada53119d841 into 445cc0befa8b471fb3a55b8b7ca1791c(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:49,248 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:49,248 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/C, priority=13, startTime=1731808368660; duration=0sec 2024-11-17T01:52:49,248 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:49,248 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:C 2024-11-17T01:52:49,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:49,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44270 deadline: 1731808429414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:49,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:49,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44298 deadline: 1731808429414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:49,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:49,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44294 deadline: 1731808429424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:49,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/11876b0ad9104e9a8de810af17707933 2024-11-17T01:52:49,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/83135d38d0f343b48def8fceb449812b is 50, key is test_row_0/B:col10/1731808369057/Put/seqid=0 2024-11-17T01:52:49,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742065_1241 (size=12001) 2024-11-17T01:52:49,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44270 deadline: 1731808429724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:49,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44298 deadline: 1731808429724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:49,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:49,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44294 deadline: 1731808429734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T01:52:49,774 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:49,774 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:52:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-17T01:52:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T01:52:49,776 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:52:49,776 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:52:49,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:52:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T01:52:49,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/83135d38d0f343b48def8fceb449812b 2024-11-17T01:52:49,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/ac4a2bdca37f4220870917c5c443cb63 is 50, key is test_row_0/C:col10/1731808369057/Put/seqid=0 2024-11-17T01:52:49,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742066_1242 (size=12001) 2024-11-17T01:52:49,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/ac4a2bdca37f4220870917c5c443cb63 2024-11-17T01:52:49,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/11876b0ad9104e9a8de810af17707933 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/11876b0ad9104e9a8de810af17707933 2024-11-17T01:52:49,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T01:52:49,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:49,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:49,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:49,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:49,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:49,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/11876b0ad9104e9a8de810af17707933, entries=200, sequenceid=97, filesize=14.0 K 2024-11-17T01:52:49,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/83135d38d0f343b48def8fceb449812b as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/83135d38d0f343b48def8fceb449812b 2024-11-17T01:52:49,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/83135d38d0f343b48def8fceb449812b, entries=150, sequenceid=97, filesize=11.7 K 2024-11-17T01:52:49,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/ac4a2bdca37f4220870917c5c443cb63 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/ac4a2bdca37f4220870917c5c443cb63 2024-11-17T01:52:49,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/ac4a2bdca37f4220870917c5c443cb63, entries=150, sequenceid=97, filesize=11.7 K 2024-11-17T01:52:49,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 36d65c1d5dfe82a6655fada53119d841 in 888ms, sequenceid=97, compaction requested=true 2024-11-17T01:52:49,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:49,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:52:49,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:49,947 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:49,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:52:49,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:49,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:52:49,947 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:49,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:49,948 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:49,948 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:49,948 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/A is initiating minor compaction (all files) 2024-11-17T01:52:49,948 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/B is initiating minor compaction (all files) 2024-11-17T01:52:49,948 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/A in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:49,948 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/B in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:49,948 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/0b34cf11592a47d89f2eb9793a96ab72, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/735702213e704c16ba6ea428958b3a5d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/11876b0ad9104e9a8de810af17707933] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=37.5 K 2024-11-17T01:52:49,948 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/63cd5dc383a341159d3f0e3b24784296, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/cbe9ec48709b4c6083a1d968cfdfa36d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/83135d38d0f343b48def8fceb449812b] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=35.3 K 2024-11-17T01:52:49,949 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b34cf11592a47d89f2eb9793a96ab72, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731808365872 2024-11-17T01:52:49,949 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 63cd5dc383a341159d3f0e3b24784296, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731808365872 2024-11-17T01:52:49,949 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 735702213e704c16ba6ea428958b3a5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731808367760 2024-11-17T01:52:49,949 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting cbe9ec48709b4c6083a1d968cfdfa36d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731808367760 2024-11-17T01:52:49,949 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11876b0ad9104e9a8de810af17707933, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731808368943 2024-11-17T01:52:49,950 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 83135d38d0f343b48def8fceb449812b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731808368943 2024-11-17T01:52:49,961 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#B#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:49,961 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/0319ffeb41f84c3e9d78ddc5f2346d15 is 50, key is test_row_0/B:col10/1731808369057/Put/seqid=0 2024-11-17T01:52:49,984 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#A#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:49,984 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/eddc848a981f4ae7a2e08abf9511b181 is 50, key is test_row_0/A:col10/1731808369057/Put/seqid=0 2024-11-17T01:52:50,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742067_1243 (size=12207) 2024-11-17T01:52:50,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742068_1244 (size=12207) 2024-11-17T01:52:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:50,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T01:52:50,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:52:50,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:50,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:52:50,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:50,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:52:50,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:50,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/1c6c31d5243640c689c51331f3f5ba43 is 50, key is test_row_0/A:col10/1731808370047/Put/seqid=0 2024-11-17T01:52:50,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742069_1245 (size=12001) 2024-11-17T01:52:50,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T01:52:50,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:50,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808430076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808430076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:50,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:50,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:50,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:50,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:50,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:50,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:50,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:50,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T01:52:50,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:50,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808430184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:50,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808430194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:50,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T01:52:50,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:50,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:50,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:50,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:50,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:50,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:50,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:50,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44298 deadline: 1731808430235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:50,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:50,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44270 deadline: 1731808430235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:50,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:50,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:50,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:50,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44294 deadline: 1731808430244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:50,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T01:52:50,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:50,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:50,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:50,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:50,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:50,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:50,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:50,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808430394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:50,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T01:52:50,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:50,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808430406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:50,409 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/eddc848a981f4ae7a2e08abf9511b181 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/eddc848a981f4ae7a2e08abf9511b181 2024-11-17T01:52:50,410 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/0319ffeb41f84c3e9d78ddc5f2346d15 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/0319ffeb41f84c3e9d78ddc5f2346d15 2024-11-17T01:52:50,418 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/B of 36d65c1d5dfe82a6655fada53119d841 into 0319ffeb41f84c3e9d78ddc5f2346d15(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:50,418 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:50,418 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/B, priority=13, startTime=1731808369947; duration=0sec 2024-11-17T01:52:50,418 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:50,418 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:B 2024-11-17T01:52:50,418 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:50,420 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:50,420 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/C is initiating minor compaction (all files) 2024-11-17T01:52:50,420 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/C in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:50,420 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/445cc0befa8b471fb3a55b8b7ca1791c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/9e86b203407545edbca37bc5b8903f78, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/ac4a2bdca37f4220870917c5c443cb63] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=35.3 K 2024-11-17T01:52:50,421 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 445cc0befa8b471fb3a55b8b7ca1791c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731808365872 2024-11-17T01:52:50,421 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e86b203407545edbca37bc5b8903f78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731808367760 2024-11-17T01:52:50,422 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting ac4a2bdca37f4220870917c5c443cb63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731808368943 2024-11-17T01:52:50,425 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/A of 36d65c1d5dfe82a6655fada53119d841 into eddc848a981f4ae7a2e08abf9511b181(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:50,425 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:50,425 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/A, priority=13, startTime=1731808369947; duration=0sec 2024-11-17T01:52:50,426 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:50,426 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:A 2024-11-17T01:52:50,431 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#C#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:50,431 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/64c14c63e22b488396acfbd99f1e8f52 is 50, key is test_row_0/C:col10/1731808369057/Put/seqid=0 2024-11-17T01:52:50,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742070_1246 (size=12207) 2024-11-17T01:52:50,451 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/64c14c63e22b488396acfbd99f1e8f52 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/64c14c63e22b488396acfbd99f1e8f52 2024-11-17T01:52:50,459 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/C of 36d65c1d5dfe82a6655fada53119d841 into 64c14c63e22b488396acfbd99f1e8f52(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:50,460 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:50,460 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/C, priority=13, startTime=1731808369947; duration=0sec 2024-11-17T01:52:50,460 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:50,460 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:C 2024-11-17T01:52:50,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/1c6c31d5243640c689c51331f3f5ba43 2024-11-17T01:52:50,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/bc96d0d4551649539b2569fcdd420abd is 50, key is test_row_0/B:col10/1731808370047/Put/seqid=0 2024-11-17T01:52:50,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742071_1247 (size=12001) 2024-11-17T01:52:50,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T01:52:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:50,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:50,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T01:52:50,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:50,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:50,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:50,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:50,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:50,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808430704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:50,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:50,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808430714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:50,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:50,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:50,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T01:52:50,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:50,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:50,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:50,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:50,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:50,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/bc96d0d4551649539b2569fcdd420abd 2024-11-17T01:52:50,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/d11f78a09f5244e4875d330dc40ba5cd is 50, key is test_row_0/C:col10/1731808370047/Put/seqid=0 2024-11-17T01:52:50,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742072_1248 (size=12001) 2024-11-17T01:52:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T01:52:51,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T01:52:51,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:51,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:51,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:51,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:51,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:51,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:51,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T01:52:51,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:51,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:51,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:51,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:51,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:51,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808431214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:51,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:51,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:51,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:51,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808431224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:51,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44270 deadline: 1731808431244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:51,248 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in 36d65c1d5dfe82a6655fada53119d841 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:52:51,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:51,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:51,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:51,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:51,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44298 deadline: 1731808431254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44294 deadline: 1731808431254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:51,258 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_2' in 36d65c1d5dfe82a6655fada53119d841 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:52:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:51,258 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in 36d65c1d5dfe82a6655fada53119d841 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:52:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:51,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/d11f78a09f5244e4875d330dc40ba5cd 2024-11-17T01:52:51,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T01:52:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:51,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:51,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/1c6c31d5243640c689c51331f3f5ba43 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/1c6c31d5243640c689c51331f3f5ba43 2024-11-17T01:52:51,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/1c6c31d5243640c689c51331f3f5ba43, entries=150, sequenceid=117, filesize=11.7 K 2024-11-17T01:52:51,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/bc96d0d4551649539b2569fcdd420abd as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/bc96d0d4551649539b2569fcdd420abd 2024-11-17T01:52:51,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/bc96d0d4551649539b2569fcdd420abd, entries=150, sequenceid=117, filesize=11.7 K 2024-11-17T01:52:51,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/d11f78a09f5244e4875d330dc40ba5cd as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/d11f78a09f5244e4875d330dc40ba5cd 2024-11-17T01:52:51,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/d11f78a09f5244e4875d330dc40ba5cd, entries=150, sequenceid=117, filesize=11.7 K 2024-11-17T01:52:51,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 36d65c1d5dfe82a6655fada53119d841 in 1287ms, sequenceid=117, compaction requested=false 2024-11-17T01:52:51,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:51,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T01:52:51,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:51,461 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T01:52:51,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:52:51,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:51,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:52:51,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:51,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:52:51,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:51,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/d7eb353de3514ddea32ac03e440950ee is 50, key is test_row_0/A:col10/1731808370074/Put/seqid=0 2024-11-17T01:52:51,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742073_1249 (size=12051) 2024-11-17T01:52:51,507 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/d7eb353de3514ddea32ac03e440950ee 2024-11-17T01:52:51,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/ed8eaeeb10844fe38983aa3597e94376 is 50, key is test_row_0/B:col10/1731808370074/Put/seqid=0 2024-11-17T01:52:51,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742074_1250 (size=12051) 2024-11-17T01:52:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T01:52:51,957 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/ed8eaeeb10844fe38983aa3597e94376 2024-11-17T01:52:51,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/270e11e9b475434ca4138ed5e042b26c is 50, key is test_row_0/C:col10/1731808370074/Put/seqid=0 2024-11-17T01:52:52,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742075_1251 (size=12051) 2024-11-17T01:52:52,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:52,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808432276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:52,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808432277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:52,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808432385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:52,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:52,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808432385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:52,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:52,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:52,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:52,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:52,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:52,423 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/270e11e9b475434ca4138ed5e042b26c 2024-11-17T01:52:52,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/d7eb353de3514ddea32ac03e440950ee as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/d7eb353de3514ddea32ac03e440950ee 2024-11-17T01:52:52,435 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/d7eb353de3514ddea32ac03e440950ee, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T01:52:52,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/ed8eaeeb10844fe38983aa3597e94376 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/ed8eaeeb10844fe38983aa3597e94376 2024-11-17T01:52:52,446 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/ed8eaeeb10844fe38983aa3597e94376, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T01:52:52,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/270e11e9b475434ca4138ed5e042b26c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/270e11e9b475434ca4138ed5e042b26c 2024-11-17T01:52:52,453 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/270e11e9b475434ca4138ed5e042b26c, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T01:52:52,455 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 36d65c1d5dfe82a6655fada53119d841 in 994ms, sequenceid=133, compaction requested=true 2024-11-17T01:52:52,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:52,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:52,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-17T01:52:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-17T01:52:52,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-17T01:52:52,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6800 sec 2024-11-17T01:52:52,462 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.6860 sec 2024-11-17T01:52:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:52,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T01:52:52,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:52:52,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:52,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:52:52,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:52,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:52:52,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:52,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/5343d305cb164ef3ae3d7a33dcb7e163 is 50, key is test_row_0/A:col10/1731808372595/Put/seqid=0 2024-11-17T01:52:52,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742076_1252 (size=14541) 2024-11-17T01:52:52,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808432616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:52,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:52,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:52,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:52,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808432620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:52,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808432724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:52,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:52,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:52,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:52,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808432734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:52,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:52,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:52,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:52,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808432934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:52,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808432944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:52,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:52,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:52,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:53,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/5343d305cb164ef3ae3d7a33dcb7e163 2024-11-17T01:52:53,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/67bc8661f6684410a304834847c05d8f is 50, key is test_row_0/B:col10/1731808372595/Put/seqid=0 2024-11-17T01:52:53,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742077_1253 (size=12151) 2024-11-17T01:52:53,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:53,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808433244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:53,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:53,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:53,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:53,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:53,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808433254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:53,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:53,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:53,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:53,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:53,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44270 deadline: 1731808433264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:53,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:53,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44294 deadline: 1731808433264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:53,266 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in 36d65c1d5dfe82a6655fada53119d841 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:52:53,266 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in 36d65c1d5dfe82a6655fada53119d841 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:52:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:53,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:53,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:53,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44298 deadline: 1731808433285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:53,288 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_2' in 36d65c1d5dfe82a6655fada53119d841 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:52:53,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:53,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:53,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:53,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/67bc8661f6684410a304834847c05d8f 2024-11-17T01:52:53,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/15bee743d3ef4105a112a6cd861f10dc is 50, key is test_row_0/C:col10/1731808372595/Put/seqid=0 2024-11-17T01:52:53,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742078_1254 (size=12151) 2024-11-17T01:52:53,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/15bee743d3ef4105a112a6cd861f10dc 2024-11-17T01:52:53,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/5343d305cb164ef3ae3d7a33dcb7e163 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/5343d305cb164ef3ae3d7a33dcb7e163 2024-11-17T01:52:53,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/5343d305cb164ef3ae3d7a33dcb7e163, entries=200, sequenceid=153, filesize=14.2 K 2024-11-17T01:52:53,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/67bc8661f6684410a304834847c05d8f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/67bc8661f6684410a304834847c05d8f 2024-11-17T01:52:53,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/67bc8661f6684410a304834847c05d8f, entries=150, sequenceid=153, filesize=11.9 K 2024-11-17T01:52:53,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/15bee743d3ef4105a112a6cd861f10dc as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/15bee743d3ef4105a112a6cd861f10dc 2024-11-17T01:52:53,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/15bee743d3ef4105a112a6cd861f10dc, entries=150, sequenceid=153, filesize=11.9 K 2024-11-17T01:52:53,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 36d65c1d5dfe82a6655fada53119d841 in 874ms, sequenceid=153, compaction requested=true 2024-11-17T01:52:53,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:53,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:52:53,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:53,470 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:52:53,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:52:53,470 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:52:53,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:53,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:52:53,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:53,471 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:52:53,471 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50800 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:52:53,472 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/A is initiating minor compaction (all files) 2024-11-17T01:52:53,472 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/B is initiating minor compaction (all files) 2024-11-17T01:52:53,472 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/A in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:53,472 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/B in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:53,472 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/0319ffeb41f84c3e9d78ddc5f2346d15, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/bc96d0d4551649539b2569fcdd420abd, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/ed8eaeeb10844fe38983aa3597e94376, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/67bc8661f6684410a304834847c05d8f] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=47.3 K 2024-11-17T01:52:53,472 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/eddc848a981f4ae7a2e08abf9511b181, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/1c6c31d5243640c689c51331f3f5ba43, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/d7eb353de3514ddea32ac03e440950ee, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/5343d305cb164ef3ae3d7a33dcb7e163] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=49.6 K 2024-11-17T01:52:53,473 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 0319ffeb41f84c3e9d78ddc5f2346d15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731808368943 2024-11-17T01:52:53,473 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting eddc848a981f4ae7a2e08abf9511b181, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731808368943 2024-11-17T01:52:53,473 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting bc96d0d4551649539b2569fcdd420abd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731808369078 2024-11-17T01:52:53,473 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c6c31d5243640c689c51331f3f5ba43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731808369078 2024-11-17T01:52:53,473 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7eb353de3514ddea32ac03e440950ee, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731808370057 2024-11-17T01:52:53,473 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting ed8eaeeb10844fe38983aa3597e94376, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731808370057 2024-11-17T01:52:53,474 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5343d305cb164ef3ae3d7a33dcb7e163, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731808372253 2024-11-17T01:52:53,474 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 67bc8661f6684410a304834847c05d8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731808372274 2024-11-17T01:52:53,484 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#A#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:53,485 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/b8a09ddf78484f32b952224a65371bc3 is 50, key is test_row_0/A:col10/1731808372595/Put/seqid=0 2024-11-17T01:52:53,486 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#B#compaction#213 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:53,487 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/cef21c9b950d4886ba8f297e7ae3452e is 50, key is test_row_0/B:col10/1731808372595/Put/seqid=0 2024-11-17T01:52:53,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742079_1255 (size=12493) 2024-11-17T01:52:53,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742080_1256 (size=12493) 2024-11-17T01:52:53,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:53,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T01:52:53,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:52:53,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:53,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:52:53,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:53,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:52:53,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:53,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/e2d76511f4254e598657c85525f05072 is 50, key is test_row_0/A:col10/1731808373756/Put/seqid=0 2024-11-17T01:52:53,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742081_1257 (size=14541) 2024-11-17T01:52:53,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/e2d76511f4254e598657c85525f05072 2024-11-17T01:52:53,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/98f0bc29bb8d4151ac20d45a9c28c723 is 50, key is test_row_0/B:col10/1731808373756/Put/seqid=0 2024-11-17T01:52:53,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742082_1258 (size=12151) 2024-11-17T01:52:53,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/98f0bc29bb8d4151ac20d45a9c28c723 2024-11-17T01:52:53,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/4f31da9d504244d7afca131db420bdd8 is 50, key is test_row_0/C:col10/1731808373756/Put/seqid=0 2024-11-17T01:52:53,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808433836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:53,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808433839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:53,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742083_1259 (size=12151) 2024-11-17T01:52:53,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/4f31da9d504244d7afca131db420bdd8 2024-11-17T01:52:53,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/e2d76511f4254e598657c85525f05072 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/e2d76511f4254e598657c85525f05072 2024-11-17T01:52:53,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/e2d76511f4254e598657c85525f05072, entries=200, sequenceid=166, filesize=14.2 K 2024-11-17T01:52:53,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/98f0bc29bb8d4151ac20d45a9c28c723 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/98f0bc29bb8d4151ac20d45a9c28c723 2024-11-17T01:52:53,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/98f0bc29bb8d4151ac20d45a9c28c723, entries=150, sequenceid=166, filesize=11.9 K 2024-11-17T01:52:53,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/4f31da9d504244d7afca131db420bdd8 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/4f31da9d504244d7afca131db420bdd8 2024-11-17T01:52:53,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/4f31da9d504244d7afca131db420bdd8, entries=150, sequenceid=166, filesize=11.9 K 2024-11-17T01:52:53,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 36d65c1d5dfe82a6655fada53119d841 in 120ms, sequenceid=166, compaction requested=true 2024-11-17T01:52:53,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:53,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:A, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:52:53,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T01:52:53,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:B, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:52:53,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-17T01:52:53,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:52:53,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-17T01:52:53,923 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/b8a09ddf78484f32b952224a65371bc3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/b8a09ddf78484f32b952224a65371bc3 2024-11-17T01:52:53,941 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/A of 36d65c1d5dfe82a6655fada53119d841 into b8a09ddf78484f32b952224a65371bc3(size=12.2 K), total size for store is 26.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:53,941 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:53,941 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/A, priority=12, startTime=1731808373470; duration=0sec 2024-11-17T01:52:53,941 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-17T01:52:53,941 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:A 2024-11-17T01:52:53,941 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:A 2024-11-17T01:52:53,941 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-17T01:52:53,943 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/cef21c9b950d4886ba8f297e7ae3452e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/cef21c9b950d4886ba8f297e7ae3452e 2024-11-17T01:52:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T01:52:53,944 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:53,944 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:52:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-17T01:52:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T01:52:53,946 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:52:53,948 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:52:53,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:52:53,952 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/B of 36d65c1d5dfe82a6655fada53119d841 into cef21c9b950d4886ba8f297e7ae3452e(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:53,952 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:53,952 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/B, priority=12, startTime=1731808373470; duration=0sec 2024-11-17T01:52:53,952 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60561 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-17T01:52:53,952 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-17T01:52:53,953 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:B 2024-11-17T01:52:53,953 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/C is initiating minor compaction (all files) 2024-11-17T01:52:53,953 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:B 2024-11-17T01:52:53,953 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/C in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:53,953 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-17T01:52:53,953 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-17T01:52:53,953 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-17T01:52:53,953 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/64c14c63e22b488396acfbd99f1e8f52, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/d11f78a09f5244e4875d330dc40ba5cd, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/270e11e9b475434ca4138ed5e042b26c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/15bee743d3ef4105a112a6cd861f10dc, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/4f31da9d504244d7afca131db420bdd8] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=59.1 K 2024-11-17T01:52:53,953 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. because compaction request was cancelled 2024-11-17T01:52:53,953 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:C 2024-11-17T01:52:53,953 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-17T01:52:53,955 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64c14c63e22b488396acfbd99f1e8f52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731808368943 2024-11-17T01:52:53,955 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-17T01:52:53,955 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-17T01:52:53,955 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. because compaction request was cancelled 2024-11-17T01:52:53,955 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:B 2024-11-17T01:52:53,955 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-17T01:52:53,955 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting d11f78a09f5244e4875d330dc40ba5cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731808369078 2024-11-17T01:52:53,956 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 270e11e9b475434ca4138ed5e042b26c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731808370057 2024-11-17T01:52:53,956 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-17T01:52:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:53,956 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-17T01:52:53,956 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. because compaction request was cancelled 2024-11-17T01:52:53,956 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:A 2024-11-17T01:52:53,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T01:52:53,956 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15bee743d3ef4105a112a6cd861f10dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731808372274 2024-11-17T01:52:53,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:52:53,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:53,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:52:53,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:53,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:52:53,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:53,957 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f31da9d504244d7afca131db420bdd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731808372614 2024-11-17T01:52:53,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/ca2d323d23074b18b67dc858742514a5 is 50, key is test_row_0/A:col10/1731808373834/Put/seqid=0 2024-11-17T01:52:53,976 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#C#compaction#219 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:53,978 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/f6310d656beb4be2bc390264c5204f2a is 50, key is test_row_0/C:col10/1731808373756/Put/seqid=0 2024-11-17T01:52:53,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742084_1260 (size=14541) 2024-11-17T01:52:53,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/ca2d323d23074b18b67dc858742514a5 2024-11-17T01:52:53,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808433987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:53,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808433992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:53,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:53,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:53,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:54,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/cc252ffbe7c047c48cc53c434294144d is 50, key is test_row_0/B:col10/1731808373834/Put/seqid=0 2024-11-17T01:52:54,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742085_1261 (size=12527) 2024-11-17T01:52:54,023 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/f6310d656beb4be2bc390264c5204f2a as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/f6310d656beb4be2bc390264c5204f2a 2024-11-17T01:52:54,031 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/C of 36d65c1d5dfe82a6655fada53119d841 into f6310d656beb4be2bc390264c5204f2a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:54,031 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:54,031 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/C, priority=11, startTime=1731808373878; duration=0sec 2024-11-17T01:52:54,031 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:54,031 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:C 2024-11-17T01:52:54,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742086_1262 (size=12151) 2024-11-17T01:52:54,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/cc252ffbe7c047c48cc53c434294144d 2024-11-17T01:52:54,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/f21ce9a578b64533b86d001c3cd70398 is 50, key is test_row_0/C:col10/1731808373834/Put/seqid=0 2024-11-17T01:52:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T01:52:54,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742087_1263 (size=12151) 2024-11-17T01:52:54,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/f21ce9a578b64533b86d001c3cd70398 2024-11-17T01:52:54,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/ca2d323d23074b18b67dc858742514a5 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/ca2d323d23074b18b67dc858742514a5 2024-11-17T01:52:54,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/ca2d323d23074b18b67dc858742514a5, entries=200, sequenceid=188, filesize=14.2 K 2024-11-17T01:52:54,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/cc252ffbe7c047c48cc53c434294144d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/cc252ffbe7c047c48cc53c434294144d 2024-11-17T01:52:54,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/cc252ffbe7c047c48cc53c434294144d, entries=150, sequenceid=188, filesize=11.9 K 2024-11-17T01:52:54,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/f21ce9a578b64533b86d001c3cd70398 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/f21ce9a578b64533b86d001c3cd70398 2024-11-17T01:52:54,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/f21ce9a578b64533b86d001c3cd70398, entries=150, sequenceid=188, filesize=11.9 K 2024-11-17T01:52:54,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 36d65c1d5dfe82a6655fada53119d841 in 142ms, sequenceid=188, compaction requested=true 2024-11-17T01:52:54,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:54,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:52:54,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:54,098 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:54,098 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:52:54,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:52:54,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:54,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:52:54,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:54,100 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:54,100 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/A is initiating minor compaction (all files) 2024-11-17T01:52:54,100 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/A in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:54,100 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/b8a09ddf78484f32b952224a65371bc3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/e2d76511f4254e598657c85525f05072, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/ca2d323d23074b18b67dc858742514a5] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=40.6 K 2024-11-17T01:52:54,100 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:52:54,100 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/B is initiating minor compaction (all files) 2024-11-17T01:52:54,100 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/B in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:54,101 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/cef21c9b950d4886ba8f297e7ae3452e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/98f0bc29bb8d4151ac20d45a9c28c723, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/cc252ffbe7c047c48cc53c434294144d] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=35.9 K 2024-11-17T01:52:54,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T01:52:54,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:54,101 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T01:52:54,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:52:54,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:54,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:52:54,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:54,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:52:54,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:54,102 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting cef21c9b950d4886ba8f297e7ae3452e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731808372274 2024-11-17T01:52:54,103 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8a09ddf78484f32b952224a65371bc3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731808372274 2024-11-17T01:52:54,106 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2d76511f4254e598657c85525f05072, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731808372602 2024-11-17T01:52:54,106 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 98f0bc29bb8d4151ac20d45a9c28c723, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731808372614 2024-11-17T01:52:54,106 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting cc252ffbe7c047c48cc53c434294144d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731808373834 2024-11-17T01:52:54,107 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca2d323d23074b18b67dc858742514a5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731808373827 2024-11-17T01:52:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:54,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:54,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/ab62d5f28ddd4a368201792e279deb64 is 50, key is test_row_0/A:col10/1731808373968/Put/seqid=0 2024-11-17T01:52:54,120 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#A#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:54,121 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/6de3fde55d5849fabdbfb0c7e6f0e8d3 is 50, key is test_row_0/A:col10/1731808373834/Put/seqid=0 2024-11-17T01:52:54,133 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#B#compaction#224 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:54,134 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/a0dd3f5c3d5449ee867fb57e4e1bf142 is 50, key is test_row_0/B:col10/1731808373834/Put/seqid=0 2024-11-17T01:52:54,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742089_1265 (size=12595) 2024-11-17T01:52:54,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742088_1264 (size=12151) 2024-11-17T01:52:54,154 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/ab62d5f28ddd4a368201792e279deb64 2024-11-17T01:52:54,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808434164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:54,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742090_1266 (size=12595) 2024-11-17T01:52:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:54,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808434169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:54,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/45c095eec5fe46ef92c967b12586de8a is 50, key is test_row_0/B:col10/1731808373968/Put/seqid=0 2024-11-17T01:52:54,186 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/a0dd3f5c3d5449ee867fb57e4e1bf142 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/a0dd3f5c3d5449ee867fb57e4e1bf142 2024-11-17T01:52:54,195 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/B of 36d65c1d5dfe82a6655fada53119d841 into a0dd3f5c3d5449ee867fb57e4e1bf142(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:54,195 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:54,195 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/B, priority=13, startTime=1731808374098; duration=0sec 2024-11-17T01:52:54,195 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:54,195 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:B 2024-11-17T01:52:54,195 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-17T01:52:54,198 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-17T01:52:54,198 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-17T01:52:54,198 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. because compaction request was cancelled 2024-11-17T01:52:54,198 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:C 2024-11-17T01:52:54,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742091_1267 (size=12151) 2024-11-17T01:52:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T01:52:54,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808434274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:54,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:54,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808434284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:54,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808434484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:54,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808434493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:54,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:54,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:54,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:54,557 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/6de3fde55d5849fabdbfb0c7e6f0e8d3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/6de3fde55d5849fabdbfb0c7e6f0e8d3 2024-11-17T01:52:54,563 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/A of 36d65c1d5dfe82a6655fada53119d841 into 6de3fde55d5849fabdbfb0c7e6f0e8d3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:54,563 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:54,563 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/A, priority=13, startTime=1731808374098; duration=0sec 2024-11-17T01:52:54,563 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:54,563 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:A 2024-11-17T01:52:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T01:52:54,603 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/45c095eec5fe46ef92c967b12586de8a 2024-11-17T01:52:54,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/e07a4767716a41e49aed46909fba1158 is 50, key is test_row_0/C:col10/1731808373968/Put/seqid=0 2024-11-17T01:52:54,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742092_1268 (size=12151) 2024-11-17T01:52:54,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808434794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:54,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808434804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:54,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:54,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:54,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:55,038 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/e07a4767716a41e49aed46909fba1158 2024-11-17T01:52:55,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/ab62d5f28ddd4a368201792e279deb64 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/ab62d5f28ddd4a368201792e279deb64 2024-11-17T01:52:55,048 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/ab62d5f28ddd4a368201792e279deb64, entries=150, sequenceid=201, filesize=11.9 K 2024-11-17T01:52:55,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/45c095eec5fe46ef92c967b12586de8a as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/45c095eec5fe46ef92c967b12586de8a 2024-11-17T01:52:55,053 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/45c095eec5fe46ef92c967b12586de8a, entries=150, sequenceid=201, filesize=11.9 K 2024-11-17T01:52:55,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/e07a4767716a41e49aed46909fba1158 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e07a4767716a41e49aed46909fba1158 2024-11-17T01:52:55,059 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e07a4767716a41e49aed46909fba1158, entries=150, sequenceid=201, filesize=11.9 K 2024-11-17T01:52:55,060 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 36d65c1d5dfe82a6655fada53119d841 in 959ms, sequenceid=201, compaction requested=true 2024-11-17T01:52:55,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:55,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:55,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-17T01:52:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-17T01:52:55,065 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-17T01:52:55,065 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1140 sec 2024-11-17T01:52:55,067 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.1220 sec 2024-11-17T01:52:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T01:52:55,084 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:55,085 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:52:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-17T01:52:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T01:52:55,087 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:52:55,088 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:52:55,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:52:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T01:52:55,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T01:52:55,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:55,241 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T01:52:55,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:52:55,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:55,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:52:55,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:55,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:52:55,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/9f2361a57571476ea2ddd45ed79c4277 is 50, key is test_row_0/A:col10/1731808374162/Put/seqid=0 2024-11-17T01:52:55,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742093_1269 (size=12151) 2024-11-17T01:52:55,273 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/9f2361a57571476ea2ddd45ed79c4277 2024-11-17T01:52:55,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/71efd0893b034109ae865e076ef4f5d4 is 50, key is test_row_0/B:col10/1731808374162/Put/seqid=0 2024-11-17T01:52:55,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742094_1270 (size=12151) 2024-11-17T01:52:55,296 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/71efd0893b034109ae865e076ef4f5d4 2024-11-17T01:52:55,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/2fc0673aad6444cc84c21a7c13c18de3 is 50, key is test_row_0/C:col10/1731808374162/Put/seqid=0 2024-11-17T01:52:55,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:55,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:55,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742095_1271 (size=12151) 2024-11-17T01:52:55,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:55,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808435335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:55,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808435338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:55,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T01:52:55,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808435444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:55,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:55,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:55,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808435454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:55,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808435654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:55,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808435664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:55,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:55,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:55,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T01:52:55,735 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/2fc0673aad6444cc84c21a7c13c18de3 2024-11-17T01:52:55,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/9f2361a57571476ea2ddd45ed79c4277 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/9f2361a57571476ea2ddd45ed79c4277 2024-11-17T01:52:55,746 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/9f2361a57571476ea2ddd45ed79c4277, entries=150, sequenceid=223, filesize=11.9 K 2024-11-17T01:52:55,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/71efd0893b034109ae865e076ef4f5d4 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/71efd0893b034109ae865e076ef4f5d4 2024-11-17T01:52:55,752 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/71efd0893b034109ae865e076ef4f5d4, entries=150, sequenceid=223, filesize=11.9 K 2024-11-17T01:52:55,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/2fc0673aad6444cc84c21a7c13c18de3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/2fc0673aad6444cc84c21a7c13c18de3 2024-11-17T01:52:55,759 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/2fc0673aad6444cc84c21a7c13c18de3, entries=150, sequenceid=223, filesize=11.9 K 2024-11-17T01:52:55,761 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 36d65c1d5dfe82a6655fada53119d841 in 520ms, sequenceid=223, compaction requested=true 2024-11-17T01:52:55,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:55,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:55,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-17T01:52:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-17T01:52:55,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-17T01:52:55,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 675 msec 2024-11-17T01:52:55,768 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 681 msec 2024-11-17T01:52:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:55,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T01:52:55,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:52:55,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:55,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:52:55,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:55,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:52:55,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:55,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/2e30136929374096b1bb357bbc3d6094 is 50, key is test_row_0/A:col10/1731808375965/Put/seqid=0 2024-11-17T01:52:55,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742096_1272 (size=14541) 2024-11-17T01:52:55,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/2e30136929374096b1bb357bbc3d6094 2024-11-17T01:52:55,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/282662365084434599834077d64335cd is 50, key is test_row_0/B:col10/1731808375965/Put/seqid=0 2024-11-17T01:52:55,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742097_1273 (size=12151) 2024-11-17T01:52:56,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808436019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:56,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808436024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:56,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808436134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:56,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808436134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T01:52:56,224 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:56,225 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:52:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-17T01:52:56,227 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:52:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T01:52:56,228 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:52:56,228 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:52:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T01:52:56,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808436344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:56,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808436344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:56,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T01:52:56,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:56,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:56,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:56,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:56,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:56,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:56,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/282662365084434599834077d64335cd 2024-11-17T01:52:56,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/24aae12665e440c69dfef5552b6073fa is 50, key is test_row_0/C:col10/1731808375965/Put/seqid=0 2024-11-17T01:52:56,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742098_1274 (size=12151) 2024-11-17T01:52:56,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/24aae12665e440c69dfef5552b6073fa 2024-11-17T01:52:56,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/2e30136929374096b1bb357bbc3d6094 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/2e30136929374096b1bb357bbc3d6094 2024-11-17T01:52:56,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/2e30136929374096b1bb357bbc3d6094, entries=200, sequenceid=236, filesize=14.2 K 2024-11-17T01:52:56,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/282662365084434599834077d64335cd as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/282662365084434599834077d64335cd 2024-11-17T01:52:56,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/282662365084434599834077d64335cd, entries=150, sequenceid=236, filesize=11.9 K 2024-11-17T01:52:56,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/24aae12665e440c69dfef5552b6073fa as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/24aae12665e440c69dfef5552b6073fa 2024-11-17T01:52:56,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/24aae12665e440c69dfef5552b6073fa, entries=150, sequenceid=236, filesize=11.9 K 2024-11-17T01:52:56,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 36d65c1d5dfe82a6655fada53119d841 in 523ms, sequenceid=236, compaction requested=true 2024-11-17T01:52:56,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:56,490 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:52:56,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:52:56,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:56,491 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:52:56,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:52:56,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:56,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:52:56,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:56,497 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:52:56,497 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/B is initiating minor compaction (all files) 2024-11-17T01:52:56,497 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/B in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:56,497 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/a0dd3f5c3d5449ee867fb57e4e1bf142, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/45c095eec5fe46ef92c967b12586de8a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/71efd0893b034109ae865e076ef4f5d4, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/282662365084434599834077d64335cd] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=47.9 K 2024-11-17T01:52:56,498 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting a0dd3f5c3d5449ee867fb57e4e1bf142, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731808373834 2024-11-17T01:52:56,498 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 45c095eec5fe46ef92c967b12586de8a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731808373968 2024-11-17T01:52:56,499 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51438 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:52:56,499 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 71efd0893b034109ae865e076ef4f5d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731808374123 2024-11-17T01:52:56,499 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/A is initiating minor compaction (all files) 2024-11-17T01:52:56,499 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/A in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:56,499 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 282662365084434599834077d64335cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731808375331 2024-11-17T01:52:56,499 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/6de3fde55d5849fabdbfb0c7e6f0e8d3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/ab62d5f28ddd4a368201792e279deb64, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/9f2361a57571476ea2ddd45ed79c4277, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/2e30136929374096b1bb357bbc3d6094] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=50.2 K 2024-11-17T01:52:56,500 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6de3fde55d5849fabdbfb0c7e6f0e8d3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731808373834 2024-11-17T01:52:56,505 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab62d5f28ddd4a368201792e279deb64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731808373968 2024-11-17T01:52:56,507 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f2361a57571476ea2ddd45ed79c4277, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731808374123 2024-11-17T01:52:56,508 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e30136929374096b1bb357bbc3d6094, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731808375331 2024-11-17T01:52:56,530 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#B#compaction#233 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:56,531 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/a2e7232a0aee4af396873f7eba114813 is 50, key is test_row_0/B:col10/1731808375965/Put/seqid=0 2024-11-17T01:52:56,539 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#A#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:56,539 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/7e81d40cae9d4e7a899ccd438f71969e is 50, key is test_row_0/A:col10/1731808375965/Put/seqid=0 2024-11-17T01:52:56,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T01:52:56,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:56,541 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T01:52:56,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:52:56,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:56,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:52:56,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:56,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:52:56,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T01:52:56,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742099_1275 (size=12731) 2024-11-17T01:52:56,584 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/a2e7232a0aee4af396873f7eba114813 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/a2e7232a0aee4af396873f7eba114813 2024-11-17T01:52:56,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/f192449fbd30467f8da19b6576180ebd is 50, key is test_row_0/A:col10/1731808376022/Put/seqid=0 2024-11-17T01:52:56,593 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/B of 36d65c1d5dfe82a6655fada53119d841 into a2e7232a0aee4af396873f7eba114813(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:56,593 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:56,593 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/B, priority=12, startTime=1731808376491; duration=0sec 2024-11-17T01:52:56,593 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:56,593 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:B 2024-11-17T01:52:56,593 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-17T01:52:56,597 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61131 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-17T01:52:56,597 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/C is initiating minor compaction (all files) 2024-11-17T01:52:56,597 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/C in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:56,597 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/f6310d656beb4be2bc390264c5204f2a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/f21ce9a578b64533b86d001c3cd70398, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e07a4767716a41e49aed46909fba1158, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/2fc0673aad6444cc84c21a7c13c18de3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/24aae12665e440c69dfef5552b6073fa] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=59.7 K 2024-11-17T01:52:56,598 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting f6310d656beb4be2bc390264c5204f2a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731808372614 2024-11-17T01:52:56,598 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting f21ce9a578b64533b86d001c3cd70398, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731808373834 2024-11-17T01:52:56,599 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting e07a4767716a41e49aed46909fba1158, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731808373968 2024-11-17T01:52:56,599 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fc0673aad6444cc84c21a7c13c18de3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731808374123 2024-11-17T01:52:56,600 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 24aae12665e440c69dfef5552b6073fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731808375331 2024-11-17T01:52:56,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742101_1277 (size=12151) 2024-11-17T01:52:56,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742100_1276 (size=12731) 2024-11-17T01:52:56,611 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/7e81d40cae9d4e7a899ccd438f71969e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/7e81d40cae9d4e7a899ccd438f71969e 2024-11-17T01:52:56,620 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/A of 36d65c1d5dfe82a6655fada53119d841 into 7e81d40cae9d4e7a899ccd438f71969e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:56,620 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:56,620 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/A, priority=12, startTime=1731808376489; duration=0sec 2024-11-17T01:52:56,621 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:56,621 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:A 2024-11-17T01:52:56,621 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#C#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:56,622 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/e510773d41644e29b2b27f81a5d60935 is 50, key is test_row_0/C:col10/1731808375965/Put/seqid=0 2024-11-17T01:52:56,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742102_1278 (size=12697) 2024-11-17T01:52:56,647 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/e510773d41644e29b2b27f81a5d60935 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e510773d41644e29b2b27f81a5d60935 2024-11-17T01:52:56,654 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/C of 36d65c1d5dfe82a6655fada53119d841 into e510773d41644e29b2b27f81a5d60935(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:56,654 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:56,654 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/C, priority=11, startTime=1731808376491; duration=0sec 2024-11-17T01:52:56,655 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:56,655 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:C 2024-11-17T01:52:56,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:56,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808436674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:56,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:56,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808436676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:56,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808436784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:56,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:56,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808436785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:56,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:56,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T01:52:56,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808436994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:56,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808436994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:56,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:56,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:56,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:57,003 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/f192449fbd30467f8da19b6576180ebd 2024-11-17T01:52:57,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/c372a41148ca451eb67634ef5810c293 is 50, key is test_row_0/B:col10/1731808376022/Put/seqid=0 2024-11-17T01:52:57,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742103_1279 (size=12151) 2024-11-17T01:52:57,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44294 deadline: 1731808437284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:57,285 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in 36d65c1d5dfe82a6655fada53119d841 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:52:57,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:57,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:57,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:57,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44270 deadline: 1731808437294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:57,297 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in 36d65c1d5dfe82a6655fada53119d841 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:52:57,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:57,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:57,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:57,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808437304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:57,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:57,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:57,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:57,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808437304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:57,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:57,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:57,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:57,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44298 deadline: 1731808437324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:57,327 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_2' in 36d65c1d5dfe82a6655fada53119d841 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:52:57,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:57,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:57,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T01:52:57,433 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/c372a41148ca451eb67634ef5810c293 2024-11-17T01:52:57,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/44ea4d98b2014db283fa961546e9114c is 50, key is test_row_0/C:col10/1731808376022/Put/seqid=0 2024-11-17T01:52:57,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742104_1280 (size=12151) 2024-11-17T01:52:57,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808437814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:57,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808437814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:57,858 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/44ea4d98b2014db283fa961546e9114c 2024-11-17T01:52:57,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/f192449fbd30467f8da19b6576180ebd as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/f192449fbd30467f8da19b6576180ebd 2024-11-17T01:52:57,867 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/f192449fbd30467f8da19b6576180ebd, entries=150, sequenceid=255, filesize=11.9 K 2024-11-17T01:52:57,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/c372a41148ca451eb67634ef5810c293 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/c372a41148ca451eb67634ef5810c293 2024-11-17T01:52:57,875 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/c372a41148ca451eb67634ef5810c293, entries=150, sequenceid=255, filesize=11.9 K 2024-11-17T01:52:57,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/44ea4d98b2014db283fa961546e9114c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/44ea4d98b2014db283fa961546e9114c 2024-11-17T01:52:57,881 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/44ea4d98b2014db283fa961546e9114c, entries=150, sequenceid=255, filesize=11.9 K 2024-11-17T01:52:57,882 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 36d65c1d5dfe82a6655fada53119d841 in 1341ms, sequenceid=255, compaction requested=false 2024-11-17T01:52:57,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:57,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:57,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-17T01:52:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-17T01:52:57,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-17T01:52:57,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6550 sec 2024-11-17T01:52:57,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.6600 sec 2024-11-17T01:52:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T01:52:58,374 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:58,375 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:52:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-17T01:52:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T01:52:58,377 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:52:58,378 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:52:58,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:52:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T01:52:58,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T01:52:58,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:58,531 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T01:52:58,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:52:58,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:58,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:52:58,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:58,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:52:58,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:58,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/67464f77a6034e4e93136a4c36fc6a90 is 50, key is test_row_0/A:col10/1731808376672/Put/seqid=0 2024-11-17T01:52:58,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742105_1281 (size=12301) 2024-11-17T01:52:58,551 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/67464f77a6034e4e93136a4c36fc6a90 2024-11-17T01:52:58,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/a22ccdfc4d1f4b93958fdc16400b8449 is 50, key is test_row_0/B:col10/1731808376672/Put/seqid=0 2024-11-17T01:52:58,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742106_1282 (size=12301) 2024-11-17T01:52:58,608 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/a22ccdfc4d1f4b93958fdc16400b8449 2024-11-17T01:52:58,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/7fa14a41c22c485f92f5c72001d19006 is 50, key is test_row_0/C:col10/1731808376672/Put/seqid=0 2024-11-17T01:52:58,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742107_1283 (size=12301) 2024-11-17T01:52:58,657 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/7fa14a41c22c485f92f5c72001d19006 2024-11-17T01:52:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T01:52:58,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/67464f77a6034e4e93136a4c36fc6a90 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/67464f77a6034e4e93136a4c36fc6a90 2024-11-17T01:52:58,703 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/67464f77a6034e4e93136a4c36fc6a90, entries=150, sequenceid=271, filesize=12.0 K 2024-11-17T01:52:58,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/a22ccdfc4d1f4b93958fdc16400b8449 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/a22ccdfc4d1f4b93958fdc16400b8449 2024-11-17T01:52:58,712 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/a22ccdfc4d1f4b93958fdc16400b8449, entries=150, sequenceid=271, filesize=12.0 K 2024-11-17T01:52:58,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/7fa14a41c22c485f92f5c72001d19006 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/7fa14a41c22c485f92f5c72001d19006 2024-11-17T01:52:58,721 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/7fa14a41c22c485f92f5c72001d19006, entries=150, sequenceid=271, filesize=12.0 K 2024-11-17T01:52:58,724 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 36d65c1d5dfe82a6655fada53119d841 in 193ms, sequenceid=271, compaction requested=true 2024-11-17T01:52:58,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:58,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:58,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-17T01:52:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-17T01:52:58,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-17T01:52:58,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 347 msec 2024-11-17T01:52:58,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 354 msec 2024-11-17T01:52:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:52:58,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T01:52:58,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:52:58,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:58,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:52:58,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:58,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:52:58,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:52:58,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/fc6fe5f49e0349828ef4c15ad7a83833 is 50, key is test_row_0/A:col10/1731808378835/Put/seqid=0 2024-11-17T01:52:58,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742108_1284 (size=14741) 2024-11-17T01:52:58,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808438898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:58,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808378888, max timestamp=1731808378902Num uniques 859; 2024-11-17T01:52:58,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808378888, max timestamp=1731808378902Num uniques 859; 2024-11-17T01:52:58,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:52:58,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808378888, max timestamp=1731808378902Num uniques 859; 2024-11-17T01:52:58,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:52:58,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:52:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:59,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T01:52:59,005 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:52:59,005 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:52:59,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-17T01:52:59,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T01:52:59,009 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:52:59,010 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:52:59,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:52:59,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:59,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808439014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:59,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:59,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808439020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:59,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:59,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:59,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:59,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T01:52:59,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:59,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808439134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:59,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:59,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:59,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:59,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T01:52:59,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:59,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:59,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:59,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:59,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:59,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:59,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808439224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:59,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/fc6fe5f49e0349828ef4c15ad7a83833 2024-11-17T01:52:59,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/31e72589c2c5403196ec836e256551ef is 50, key is test_row_0/B:col10/1731808378835/Put/seqid=0 2024-11-17T01:52:59,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742109_1285 (size=12301) 2024-11-17T01:52:59,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T01:52:59,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:59,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:59,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:59,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:59,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T01:52:59,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808439344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:59,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:59,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:59,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:59,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T01:52:59,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:59,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:59,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:59,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:59,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:59,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808439535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:59,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:59,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:59,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:59,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T01:52:59,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:59,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:59,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:59,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:59,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:59,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:59,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T01:52:59,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:52:59,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808439655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:52:59,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:59,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:52:59,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:52:59,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/31e72589c2c5403196ec836e256551ef 2024-11-17T01:52:59,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/283cdc9a5946477198a3a522b73721f7 is 50, key is test_row_0/C:col10/1731808378835/Put/seqid=0 2024-11-17T01:52:59,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742110_1286 (size=12301) 2024-11-17T01:52:59,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/283cdc9a5946477198a3a522b73721f7 2024-11-17T01:52:59,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/fc6fe5f49e0349828ef4c15ad7a83833 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/fc6fe5f49e0349828ef4c15ad7a83833 2024-11-17T01:52:59,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T01:52:59,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:59,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:52:59,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:59,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:59,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:59,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:52:59,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/fc6fe5f49e0349828ef4c15ad7a83833, entries=200, sequenceid=282, filesize=14.4 K 2024-11-17T01:52:59,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/31e72589c2c5403196ec836e256551ef as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/31e72589c2c5403196ec836e256551ef 2024-11-17T01:52:59,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/31e72589c2c5403196ec836e256551ef, entries=150, sequenceid=282, filesize=12.0 K 2024-11-17T01:52:59,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/283cdc9a5946477198a3a522b73721f7 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/283cdc9a5946477198a3a522b73721f7 2024-11-17T01:52:59,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/283cdc9a5946477198a3a522b73721f7, entries=150, sequenceid=282, filesize=12.0 K 2024-11-17T01:52:59,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 36d65c1d5dfe82a6655fada53119d841 in 962ms, sequenceid=282, compaction requested=true 2024-11-17T01:52:59,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:59,801 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:52:59,804 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51924 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:52:59,804 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/A is initiating minor compaction (all files) 2024-11-17T01:52:59,804 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/A in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:59,804 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/7e81d40cae9d4e7a899ccd438f71969e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/f192449fbd30467f8da19b6576180ebd, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/67464f77a6034e4e93136a4c36fc6a90, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/fc6fe5f49e0349828ef4c15ad7a83833] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=50.7 K 2024-11-17T01:52:59,805 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e81d40cae9d4e7a899ccd438f71969e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731808375331 2024-11-17T01:52:59,805 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting f192449fbd30467f8da19b6576180ebd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731808375983 2024-11-17T01:52:59,805 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67464f77a6034e4e93136a4c36fc6a90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731808376671 2024-11-17T01:52:59,806 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc6fe5f49e0349828ef4c15ad7a83833, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731808378833 2024-11-17T01:52:59,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:52:59,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:59,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:52:59,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:52:59,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:52:59,814 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:52:59,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:59,815 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49484 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:52:59,816 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/B is initiating minor compaction (all files) 2024-11-17T01:52:59,816 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/B in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:59,816 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/a2e7232a0aee4af396873f7eba114813, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/c372a41148ca451eb67634ef5810c293, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/a22ccdfc4d1f4b93958fdc16400b8449, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/31e72589c2c5403196ec836e256551ef] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=48.3 K 2024-11-17T01:52:59,816 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting a2e7232a0aee4af396873f7eba114813, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731808375331 2024-11-17T01:52:59,817 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting c372a41148ca451eb67634ef5810c293, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731808375983 2024-11-17T01:52:59,817 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting a22ccdfc4d1f4b93958fdc16400b8449, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731808376671 2024-11-17T01:52:59,817 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 31e72589c2c5403196ec836e256551ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731808378833 2024-11-17T01:52:59,821 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#A#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:59,822 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/1dd23857dd394cb9bd32c0cfcd84b9d7 is 50, key is test_row_0/A:col10/1731808378835/Put/seqid=0 2024-11-17T01:52:59,833 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#B#compaction#246 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:59,833 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/f9734ba6e0f24403ae8e175249f1ef6f is 50, key is test_row_0/B:col10/1731808378835/Put/seqid=0 2024-11-17T01:52:59,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742112_1288 (size=13017) 2024-11-17T01:52:59,894 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/f9734ba6e0f24403ae8e175249f1ef6f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/f9734ba6e0f24403ae8e175249f1ef6f 2024-11-17T01:52:59,901 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/B of 36d65c1d5dfe82a6655fada53119d841 into f9734ba6e0f24403ae8e175249f1ef6f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:52:59,902 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:52:59,902 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/B, priority=12, startTime=1731808379814; duration=0sec 2024-11-17T01:52:59,902 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:52:59,902 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:B 2024-11-17T01:52:59,902 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:52:59,904 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:52:59,904 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/C is initiating minor compaction (all files) 2024-11-17T01:52:59,904 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/C in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:59,904 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e510773d41644e29b2b27f81a5d60935, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/44ea4d98b2014db283fa961546e9114c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/7fa14a41c22c485f92f5c72001d19006, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/283cdc9a5946477198a3a522b73721f7] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=48.3 K 2024-11-17T01:52:59,904 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting e510773d41644e29b2b27f81a5d60935, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731808375331 2024-11-17T01:52:59,904 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 44ea4d98b2014db283fa961546e9114c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731808375983 2024-11-17T01:52:59,905 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fa14a41c22c485f92f5c72001d19006, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731808376671 2024-11-17T01:52:59,906 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 283cdc9a5946477198a3a522b73721f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731808378833 2024-11-17T01:52:59,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742111_1287 (size=13017) 2024-11-17T01:52:59,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T01:52:59,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:52:59,932 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-17T01:52:59,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:52:59,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:52:59,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:52:59,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:52:59,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:52:59,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:52:59,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/cda097e985154fd19983362e1fd06ced is 50, key is test_row_0/A:col10/1731808379019/Put/seqid=0 2024-11-17T01:52:59,965 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#C#compaction#248 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:52:59,966 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/d3bf1c3ad74641a180dadb68b730ad6b is 50, key is test_row_0/C:col10/1731808378835/Put/seqid=0 2024-11-17T01:52:59,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742113_1289 (size=12301) 2024-11-17T01:52:59,998 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/cda097e985154fd19983362e1fd06ced 2024-11-17T01:53:00,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742114_1290 (size=12983) 2024-11-17T01:53:00,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/7145fbf134ce4ca183b1dd2044a8b7b5 is 50, key is test_row_0/B:col10/1731808379019/Put/seqid=0 2024-11-17T01:53:00,023 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/d3bf1c3ad74641a180dadb68b730ad6b as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/d3bf1c3ad74641a180dadb68b730ad6b 2024-11-17T01:53:00,040 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/C of 36d65c1d5dfe82a6655fada53119d841 into d3bf1c3ad74641a180dadb68b730ad6b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:00,040 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:53:00,040 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/C, priority=12, startTime=1731808379814; duration=0sec 2024-11-17T01:53:00,040 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:00,040 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:C 2024-11-17T01:53:00,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:53:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:53:00,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742115_1291 (size=12301) 2024-11-17T01:53:00,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808440115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T01:53:00,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:00,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808440165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:00,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:00,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:00,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:00,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808440224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:00,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:00,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:00,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:00,333 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/1dd23857dd394cb9bd32c0cfcd84b9d7 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/1dd23857dd394cb9bd32c0cfcd84b9d7 2024-11-17T01:53:00,340 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/A of 36d65c1d5dfe82a6655fada53119d841 into 1dd23857dd394cb9bd32c0cfcd84b9d7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:00,340 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:53:00,340 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/A, priority=12, startTime=1731808379801; duration=0sec 2024-11-17T01:53:00,340 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:00,340 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:A 2024-11-17T01:53:00,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808440434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:00,469 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/7145fbf134ce4ca183b1dd2044a8b7b5 2024-11-17T01:53:00,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/e26f044cd6bb4b4ba3358c5e646f333f is 50, key is test_row_0/C:col10/1731808379019/Put/seqid=0 2024-11-17T01:53:00,502 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T01:53:00,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742116_1292 (size=12301) 2024-11-17T01:53:00,521 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/e26f044cd6bb4b4ba3358c5e646f333f 2024-11-17T01:53:00,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/cda097e985154fd19983362e1fd06ced as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/cda097e985154fd19983362e1fd06ced 2024-11-17T01:53:00,539 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/cda097e985154fd19983362e1fd06ced, entries=150, sequenceid=319, filesize=12.0 K 2024-11-17T01:53:00,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/7145fbf134ce4ca183b1dd2044a8b7b5 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/7145fbf134ce4ca183b1dd2044a8b7b5 2024-11-17T01:53:00,549 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/7145fbf134ce4ca183b1dd2044a8b7b5, entries=150, sequenceid=319, filesize=12.0 K 2024-11-17T01:53:00,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/e26f044cd6bb4b4ba3358c5e646f333f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e26f044cd6bb4b4ba3358c5e646f333f 2024-11-17T01:53:00,557 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e26f044cd6bb4b4ba3358c5e646f333f, entries=150, sequenceid=319, filesize=12.0 K 2024-11-17T01:53:00,559 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 36d65c1d5dfe82a6655fada53119d841 in 626ms, sequenceid=319, compaction requested=false 2024-11-17T01:53:00,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:53:00,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:00,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-17T01:53:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-17T01:53:00,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-17T01:53:00,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5510 sec 2024-11-17T01:53:00,570 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.5630 sec 2024-11-17T01:53:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:53:00,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T01:53:00,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:53:00,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:00,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:53:00,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:00,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:53:00,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:00,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/e9837b9c26ca44228472a7ca643ee94b is 50, key is test_row_0/A:col10/1731808380746/Put/seqid=0 2024-11-17T01:53:00,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742117_1293 (size=14741) 2024-11-17T01:53:00,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:00,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808440821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:00,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:00,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808440934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:01,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808441144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:01,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:01,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:01,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T01:53:01,154 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:01,154 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:53:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-17T01:53:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T01:53:01,156 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:53:01,156 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:53:01,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:53:01,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/e9837b9c26ca44228472a7ca643ee94b 2024-11-17T01:53:01,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808441184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:01,186 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in 36d65c1d5dfe82a6655fada53119d841 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:53:01,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:01,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:01,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:01,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/0f94821b96d643a5a68b8530ef7cddec is 50, key is test_row_0/B:col10/1731808380746/Put/seqid=0 2024-11-17T01:53:01,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742118_1294 (size=12301) 2024-11-17T01:53:01,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/0f94821b96d643a5a68b8530ef7cddec 2024-11-17T01:53:01,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/51cd98bdcd4846a58acb2efdd5701d49 is 50, key is test_row_0/C:col10/1731808380746/Put/seqid=0 2024-11-17T01:53:01,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742119_1295 (size=12301) 2024-11-17T01:53:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T01:53:01,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T01:53:01,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:01,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:53:01,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:01,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:01,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:01,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808441454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:01,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:01,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:01,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:01,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T01:53:01,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:01,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:53:01,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:01,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:01,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T01:53:01,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T01:53:01,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:01,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:53:01,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:01,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:01,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:01,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/51cd98bdcd4846a58acb2efdd5701d49 2024-11-17T01:53:01,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/e9837b9c26ca44228472a7ca643ee94b as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/e9837b9c26ca44228472a7ca643ee94b 2024-11-17T01:53:01,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/e9837b9c26ca44228472a7ca643ee94b, entries=200, sequenceid=333, filesize=14.4 K 2024-11-17T01:53:01,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/0f94821b96d643a5a68b8530ef7cddec as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/0f94821b96d643a5a68b8530ef7cddec 2024-11-17T01:53:01,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/0f94821b96d643a5a68b8530ef7cddec, entries=150, sequenceid=333, filesize=12.0 K 2024-11-17T01:53:01,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/51cd98bdcd4846a58acb2efdd5701d49 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/51cd98bdcd4846a58acb2efdd5701d49 2024-11-17T01:53:01,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/51cd98bdcd4846a58acb2efdd5701d49, entries=150, sequenceid=333, filesize=12.0 K 2024-11-17T01:53:01,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 36d65c1d5dfe82a6655fada53119d841 in 901ms, sequenceid=333, compaction requested=true 2024-11-17T01:53:01,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:53:01,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:53:01,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:01,649 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:01,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:53:01,649 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:01,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:01,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:53:01,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:01,650 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:01,650 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:01,650 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/A is initiating minor compaction (all files) 2024-11-17T01:53:01,650 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/B is initiating minor compaction (all files) 2024-11-17T01:53:01,650 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/A in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:01,650 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/B in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:01,650 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/1dd23857dd394cb9bd32c0cfcd84b9d7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/cda097e985154fd19983362e1fd06ced, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/e9837b9c26ca44228472a7ca643ee94b] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=39.1 K 2024-11-17T01:53:01,650 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/f9734ba6e0f24403ae8e175249f1ef6f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/7145fbf134ce4ca183b1dd2044a8b7b5, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/0f94821b96d643a5a68b8530ef7cddec] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=36.7 K 2024-11-17T01:53:01,650 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dd23857dd394cb9bd32c0cfcd84b9d7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731808378833 2024-11-17T01:53:01,650 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting cda097e985154fd19983362e1fd06ced, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731808379001 2024-11-17T01:53:01,650 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting f9734ba6e0f24403ae8e175249f1ef6f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731808378833 2024-11-17T01:53:01,650 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 7145fbf134ce4ca183b1dd2044a8b7b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731808379001 2024-11-17T01:53:01,650 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9837b9c26ca44228472a7ca643ee94b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731808380088 2024-11-17T01:53:01,651 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f94821b96d643a5a68b8530ef7cddec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731808380088 2024-11-17T01:53:01,658 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#A#compaction#254 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:01,659 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#B#compaction#255 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:01,660 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/b81607b7ba07445fad723a0ef610c3c2 is 50, key is test_row_0/B:col10/1731808380746/Put/seqid=0 2024-11-17T01:53:01,663 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/b4d9c576e3c7496594ddf9028e23847e is 50, key is test_row_0/A:col10/1731808380746/Put/seqid=0 2024-11-17T01:53:01,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742120_1296 (size=13119) 2024-11-17T01:53:01,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742121_1297 (size=13119) 2024-11-17T01:53:01,676 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/b4d9c576e3c7496594ddf9028e23847e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/b4d9c576e3c7496594ddf9028e23847e 2024-11-17T01:53:01,684 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/A of 36d65c1d5dfe82a6655fada53119d841 into b4d9c576e3c7496594ddf9028e23847e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:01,684 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:53:01,684 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/A, priority=13, startTime=1731808381649; duration=0sec 2024-11-17T01:53:01,685 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:01,685 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:A 2024-11-17T01:53:01,685 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:01,686 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:01,686 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/C is initiating minor compaction (all files) 2024-11-17T01:53:01,686 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/C in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:01,686 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/d3bf1c3ad74641a180dadb68b730ad6b, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e26f044cd6bb4b4ba3358c5e646f333f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/51cd98bdcd4846a58acb2efdd5701d49] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=36.7 K 2024-11-17T01:53:01,686 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3bf1c3ad74641a180dadb68b730ad6b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731808378833 2024-11-17T01:53:01,687 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting e26f044cd6bb4b4ba3358c5e646f333f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731808379001 2024-11-17T01:53:01,687 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51cd98bdcd4846a58acb2efdd5701d49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731808380088 2024-11-17T01:53:01,695 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#C#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:01,696 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/d2b882f8fdad4e8a9b1723523922dcb5 is 50, key is test_row_0/C:col10/1731808380746/Put/seqid=0 2024-11-17T01:53:01,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742122_1298 (size=13085) 2024-11-17T01:53:01,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T01:53:01,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:01,768 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T01:53:01,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:53:01,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:01,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:53:01,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:01,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:53:01,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:01,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/c014e15ccc9745e8b7419fdf40ba5e8c is 50, key is test_row_0/A:col10/1731808380819/Put/seqid=0 2024-11-17T01:53:01,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742123_1299 (size=12301) 2024-11-17T01:53:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T01:53:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:53:01,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:53:02,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808441999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:02,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:02,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:02,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:02,074 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/b81607b7ba07445fad723a0ef610c3c2 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/b81607b7ba07445fad723a0ef610c3c2 2024-11-17T01:53:02,079 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/B of 36d65c1d5dfe82a6655fada53119d841 into b81607b7ba07445fad723a0ef610c3c2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:02,079 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:53:02,079 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/B, priority=13, startTime=1731808381649; duration=0sec 2024-11-17T01:53:02,080 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:02,080 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:B 2024-11-17T01:53:02,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808442114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:02,116 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/d2b882f8fdad4e8a9b1723523922dcb5 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/d2b882f8fdad4e8a9b1723523922dcb5 2024-11-17T01:53:02,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:02,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:02,121 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/C of 36d65c1d5dfe82a6655fada53119d841 into d2b882f8fdad4e8a9b1723523922dcb5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:02,121 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:53:02,121 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/C, priority=13, startTime=1731808381649; duration=0sec 2024-11-17T01:53:02,121 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:02,121 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:C 2024-11-17T01:53:02,176 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/c014e15ccc9745e8b7419fdf40ba5e8c 2024-11-17T01:53:02,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/33b22d9fb2044ef2a604346f69bbe723 is 50, key is test_row_0/B:col10/1731808380819/Put/seqid=0 2024-11-17T01:53:02,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742124_1300 (size=12301) 2024-11-17T01:53:02,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T01:53:02,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:02,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808442324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:02,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:02,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:02,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:02,587 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/33b22d9fb2044ef2a604346f69bbe723 2024-11-17T01:53:02,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/ad46cf7476fd4a5db369eae429f8151d is 50, key is test_row_0/C:col10/1731808380819/Put/seqid=0 2024-11-17T01:53:02,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742125_1301 (size=12301) 2024-11-17T01:53:02,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808442634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:02,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:02,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:02,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:03,007 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/ad46cf7476fd4a5db369eae429f8151d 2024-11-17T01:53:03,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/c014e15ccc9745e8b7419fdf40ba5e8c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/c014e15ccc9745e8b7419fdf40ba5e8c 2024-11-17T01:53:03,017 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/c014e15ccc9745e8b7419fdf40ba5e8c, entries=150, sequenceid=354, filesize=12.0 K 2024-11-17T01:53:03,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/33b22d9fb2044ef2a604346f69bbe723 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/33b22d9fb2044ef2a604346f69bbe723 2024-11-17T01:53:03,021 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/33b22d9fb2044ef2a604346f69bbe723, entries=150, sequenceid=354, filesize=12.0 K 2024-11-17T01:53:03,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/ad46cf7476fd4a5db369eae429f8151d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/ad46cf7476fd4a5db369eae429f8151d 2024-11-17T01:53:03,027 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/ad46cf7476fd4a5db369eae429f8151d, entries=150, sequenceid=354, filesize=12.0 K 2024-11-17T01:53:03,028 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 36d65c1d5dfe82a6655fada53119d841 in 1260ms, sequenceid=354, compaction requested=false 2024-11-17T01:53:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:53:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-17T01:53:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-17T01:53:03,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-17T01:53:03,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8730 sec 2024-11-17T01:53:03,032 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.8770 sec 2024-11-17T01:53:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:53:03,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T01:53:03,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:53:03,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:03,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:53:03,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:03,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:53:03,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:03,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/1ed9105120914068bc77d40c6cf1c8a5 is 50, key is test_row_0/A:col10/1731808383145/Put/seqid=0 2024-11-17T01:53:03,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742126_1302 (size=14741) 2024-11-17T01:53:03,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808443207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:03,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808443210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T01:53:03,304 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:03,305 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:53:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-17T01:53:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T01:53:03,307 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:53:03,308 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:53:03,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:53:03,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808443324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:03,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808443324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T01:53:03,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T01:53:03,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:03,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:53:03,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:03,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:03,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:03,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808443534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:03,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808443534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:03,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/1ed9105120914068bc77d40c6cf1c8a5 2024-11-17T01:53:03,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/02f3cf92beae4d11b5e270ea6c0ee83d is 50, key is test_row_0/B:col10/1731808383145/Put/seqid=0 2024-11-17T01:53:03,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742127_1303 (size=12301) 2024-11-17T01:53:03,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/02f3cf92beae4d11b5e270ea6c0ee83d 2024-11-17T01:53:03,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/948fd5115bab4ca8a1817157b7209d07 is 50, key is test_row_0/C:col10/1731808383145/Put/seqid=0 2024-11-17T01:53:03,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T01:53:03,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:03,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:53:03,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:03,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:03,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:03,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742128_1304 (size=12301) 2024-11-17T01:53:03,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/948fd5115bab4ca8a1817157b7209d07 2024-11-17T01:53:03,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/1ed9105120914068bc77d40c6cf1c8a5 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/1ed9105120914068bc77d40c6cf1c8a5 2024-11-17T01:53:03,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T01:53:03,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/1ed9105120914068bc77d40c6cf1c8a5, entries=200, sequenceid=369, filesize=14.4 K 2024-11-17T01:53:03,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/02f3cf92beae4d11b5e270ea6c0ee83d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/02f3cf92beae4d11b5e270ea6c0ee83d 2024-11-17T01:53:03,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/02f3cf92beae4d11b5e270ea6c0ee83d, entries=150, sequenceid=369, filesize=12.0 K 2024-11-17T01:53:03,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/948fd5115bab4ca8a1817157b7209d07 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/948fd5115bab4ca8a1817157b7209d07 2024-11-17T01:53:03,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/948fd5115bab4ca8a1817157b7209d07, entries=150, sequenceid=369, filesize=12.0 K 2024-11-17T01:53:03,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 36d65c1d5dfe82a6655fada53119d841 in 494ms, sequenceid=369, compaction requested=true 2024-11-17T01:53:03,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:53:03,640 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:03,641 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:03,641 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/A is initiating minor compaction (all files) 2024-11-17T01:53:03,641 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/A in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:03,641 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/b4d9c576e3c7496594ddf9028e23847e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/c014e15ccc9745e8b7419fdf40ba5e8c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/1ed9105120914068bc77d40c6cf1c8a5] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=39.2 K 2024-11-17T01:53:03,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:53:03,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:03,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:53:03,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:03,642 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4d9c576e3c7496594ddf9028e23847e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731808380088 2024-11-17T01:53:03,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:53:03,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:03,642 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:03,642 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting c014e15ccc9745e8b7419fdf40ba5e8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731808380811 2024-11-17T01:53:03,642 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ed9105120914068bc77d40c6cf1c8a5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731808381986 2024-11-17T01:53:03,642 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:03,642 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/B is initiating minor compaction (all files) 2024-11-17T01:53:03,643 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/B in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:03,643 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/b81607b7ba07445fad723a0ef610c3c2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/33b22d9fb2044ef2a604346f69bbe723, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/02f3cf92beae4d11b5e270ea6c0ee83d] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=36.8 K 2024-11-17T01:53:03,643 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting b81607b7ba07445fad723a0ef610c3c2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731808380088 2024-11-17T01:53:03,643 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 33b22d9fb2044ef2a604346f69bbe723, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731808380811 2024-11-17T01:53:03,643 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 02f3cf92beae4d11b5e270ea6c0ee83d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731808381986 2024-11-17T01:53:03,651 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#A#compaction#263 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:03,651 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/6beee640a5424935834c35f9f160d377 is 50, key is test_row_0/A:col10/1731808383145/Put/seqid=0 2024-11-17T01:53:03,653 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#B#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:03,653 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/b188a83ad8d1452f91399068dd4d8dc3 is 50, key is test_row_0/B:col10/1731808383145/Put/seqid=0 2024-11-17T01:53:03,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742129_1305 (size=13221) 2024-11-17T01:53:03,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742130_1306 (size=13221) 2024-11-17T01:53:03,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T01:53:03,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:03,766 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T01:53:03,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:53:03,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:03,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:53:03,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:03,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:53:03,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:03,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/b3d1a7fe459b4ab191171bef5922e725 is 50, key is test_row_0/A:col10/1731808383182/Put/seqid=0 2024-11-17T01:53:03,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742131_1307 (size=12301) 2024-11-17T01:53:03,777 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/b3d1a7fe459b4ab191171bef5922e725 2024-11-17T01:53:03,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/2aa5ea551dfe4873a3d7045de040e1f3 is 50, key is test_row_0/B:col10/1731808383182/Put/seqid=0 2024-11-17T01:53:03,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742132_1308 (size=12301) 2024-11-17T01:53:03,792 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/2aa5ea551dfe4873a3d7045de040e1f3 2024-11-17T01:53:03,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/c2486df61c014c5a8bd922fd61a64c3e is 50, key is test_row_0/C:col10/1731808383182/Put/seqid=0 2024-11-17T01:53:03,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742133_1309 (size=12301) 2024-11-17T01:53:03,822 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/c2486df61c014c5a8bd922fd61a64c3e 2024-11-17T01:53:03,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/b3d1a7fe459b4ab191171bef5922e725 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/b3d1a7fe459b4ab191171bef5922e725 2024-11-17T01:53:03,833 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/b3d1a7fe459b4ab191171bef5922e725, entries=150, sequenceid=388, filesize=12.0 K 2024-11-17T01:53:03,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/2aa5ea551dfe4873a3d7045de040e1f3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/2aa5ea551dfe4873a3d7045de040e1f3 2024-11-17T01:53:03,841 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/2aa5ea551dfe4873a3d7045de040e1f3, entries=150, sequenceid=388, filesize=12.0 K 2024-11-17T01:53:03,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/c2486df61c014c5a8bd922fd61a64c3e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/c2486df61c014c5a8bd922fd61a64c3e 2024-11-17T01:53:03,845 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/c2486df61c014c5a8bd922fd61a64c3e, entries=150, sequenceid=388, filesize=12.0 K 2024-11-17T01:53:03,846 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 36d65c1d5dfe82a6655fada53119d841 in 80ms, sequenceid=388, compaction requested=true 2024-11-17T01:53:03,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:53:03,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:03,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-17T01:53:03,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-17T01:53:03,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-17T01:53:03,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 540 msec 2024-11-17T01:53:03,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 547 msec 2024-11-17T01:53:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:53:03,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T01:53:03,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:53:03,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:03,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:53:03,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:03,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:53:03,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:03,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/bfcee8d817bc4aac996de01241779226 is 50, key is test_row_0/A:col10/1731808383857/Put/seqid=0 2024-11-17T01:53:03,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742134_1310 (size=14741) 2024-11-17T01:53:03,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808383910, max timestamp=1731808383921Num uniques 859; 2024-11-17T01:53:03,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:53:03,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808383910, max timestamp=1731808383921Num uniques 859; 2024-11-17T01:53:03,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:53:03,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808383910, max timestamp=1731808383921Num uniques 859; 2024-11-17T01:53:03,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:53:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T01:53:03,934 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:03,935 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:53:03,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-17T01:53:03,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T01:53:03,937 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:53:03,937 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:53:03,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:53:03,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:03,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808443969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:03,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:03,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808443975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:04,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T01:53:04,065 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/6beee640a5424935834c35f9f160d377 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/6beee640a5424935834c35f9f160d377 2024-11-17T01:53:04,069 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/A of 36d65c1d5dfe82a6655fada53119d841 into 6beee640a5424935834c35f9f160d377(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:04,069 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:53:04,069 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/A, priority=13, startTime=1731808383640; duration=0sec 2024-11-17T01:53:04,069 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:04,069 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:A 2024-11-17T01:53:04,069 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:53:04,072 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:53:04,072 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/C is initiating minor compaction (all files) 2024-11-17T01:53:04,072 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/C in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:04,072 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/d2b882f8fdad4e8a9b1723523922dcb5, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/ad46cf7476fd4a5db369eae429f8151d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/948fd5115bab4ca8a1817157b7209d07, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/c2486df61c014c5a8bd922fd61a64c3e] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=48.8 K 2024-11-17T01:53:04,073 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2b882f8fdad4e8a9b1723523922dcb5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731808380088 2024-11-17T01:53:04,073 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad46cf7476fd4a5db369eae429f8151d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731808380811 2024-11-17T01:53:04,073 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/b188a83ad8d1452f91399068dd4d8dc3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/b188a83ad8d1452f91399068dd4d8dc3 2024-11-17T01:53:04,073 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 948fd5115bab4ca8a1817157b7209d07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731808381986 2024-11-17T01:53:04,074 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2486df61c014c5a8bd922fd61a64c3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731808383182 2024-11-17T01:53:04,078 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/B of 36d65c1d5dfe82a6655fada53119d841 into b188a83ad8d1452f91399068dd4d8dc3(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:04,078 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:53:04,078 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/B, priority=13, startTime=1731808383642; duration=0sec 2024-11-17T01:53:04,078 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:04,078 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:B 2024-11-17T01:53:04,083 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#C#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:04,084 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/dafb90eea8784ff6a84a0665065d1dcc is 50, key is test_row_0/C:col10/1731808383182/Put/seqid=0 2024-11-17T01:53:04,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:04,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808444084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:04,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808444084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:04,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T01:53:04,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:04,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:53:04,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:04,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:04,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:04,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742135_1311 (size=13221) 2024-11-17T01:53:04,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T01:53:04,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:04,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:53:04,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:04,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:04,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:04,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:04,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T01:53:04,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/bfcee8d817bc4aac996de01241779226 2024-11-17T01:53:04,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/52b7c62f3de947e28c45c177f1f08304 is 50, key is test_row_0/B:col10/1731808383857/Put/seqid=0 2024-11-17T01:53:04,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808444294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:04,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808444294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:04,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742136_1312 (size=12301) 2024-11-17T01:53:04,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T01:53:04,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:04,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:53:04,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:04,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:04,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:04,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:04,504 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/dafb90eea8784ff6a84a0665065d1dcc as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/dafb90eea8784ff6a84a0665065d1dcc 2024-11-17T01:53:04,508 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/C of 36d65c1d5dfe82a6655fada53119d841 into dafb90eea8784ff6a84a0665065d1dcc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:04,508 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:53:04,508 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/C, priority=12, startTime=1731808383642; duration=0sec 2024-11-17T01:53:04,508 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:04,508 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:C 2024-11-17T01:53:04,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T01:53:04,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:04,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:53:04,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:04,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:04,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T01:53:04,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808444604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:04,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808444604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:04,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T01:53:04,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:04,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:53:04,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:04,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:04,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:04,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/52b7c62f3de947e28c45c177f1f08304 2024-11-17T01:53:04,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/e91dad38226048ada10e9ecc38b00cc0 is 50, key is test_row_0/C:col10/1731808383857/Put/seqid=0 2024-11-17T01:53:04,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742137_1313 (size=12301) 2024-11-17T01:53:04,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T01:53:04,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:04,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:53:04,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:04,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:04,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:04,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:05,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T01:53:05,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:05,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:53:05,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:05,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:05,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:05,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T01:53:05,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/e91dad38226048ada10e9ecc38b00cc0 2024-11-17T01:53:05,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44248 deadline: 1731808445114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:05,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:05,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1731808445114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:05,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:05,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:05,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:05,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:05,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:05,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/bfcee8d817bc4aac996de01241779226 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/bfcee8d817bc4aac996de01241779226 2024-11-17T01:53:05,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/bfcee8d817bc4aac996de01241779226, entries=200, sequenceid=398, filesize=14.4 K 2024-11-17T01:53:05,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/52b7c62f3de947e28c45c177f1f08304 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/52b7c62f3de947e28c45c177f1f08304 2024-11-17T01:53:05,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/52b7c62f3de947e28c45c177f1f08304, entries=150, sequenceid=398, filesize=12.0 K 2024-11-17T01:53:05,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/e91dad38226048ada10e9ecc38b00cc0 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e91dad38226048ada10e9ecc38b00cc0 2024-11-17T01:53:05,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e91dad38226048ada10e9ecc38b00cc0, entries=150, sequenceid=398, filesize=12.0 K 2024-11-17T01:53:05,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 36d65c1d5dfe82a6655fada53119d841 in 1263ms, sequenceid=398, compaction requested=true 2024-11-17T01:53:05,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:53:05,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:53:05,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:05,131 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:05,131 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:05,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:53:05,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:05,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36d65c1d5dfe82a6655fada53119d841:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:53:05,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:05,132 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:05,132 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:05,132 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/A is initiating minor compaction (all files) 2024-11-17T01:53:05,132 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 36d65c1d5dfe82a6655fada53119d841/B is initiating minor compaction (all files) 2024-11-17T01:53:05,132 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/A in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:05,132 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36d65c1d5dfe82a6655fada53119d841/B in TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:05,132 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/6beee640a5424935834c35f9f160d377, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/b3d1a7fe459b4ab191171bef5922e725, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/bfcee8d817bc4aac996de01241779226] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=39.3 K 2024-11-17T01:53:05,132 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/b188a83ad8d1452f91399068dd4d8dc3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/2aa5ea551dfe4873a3d7045de040e1f3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/52b7c62f3de947e28c45c177f1f08304] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp, totalSize=36.9 K 2024-11-17T01:53:05,132 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6beee640a5424935834c35f9f160d377, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731808381986 2024-11-17T01:53:05,132 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting b188a83ad8d1452f91399068dd4d8dc3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731808381986 2024-11-17T01:53:05,133 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3d1a7fe459b4ab191171bef5922e725, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731808383182 2024-11-17T01:53:05,133 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aa5ea551dfe4873a3d7045de040e1f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731808383182 2024-11-17T01:53:05,133 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfcee8d817bc4aac996de01241779226, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731808383848 2024-11-17T01:53:05,134 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 52b7c62f3de947e28c45c177f1f08304, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731808383857 2024-11-17T01:53:05,140 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#B#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:05,140 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36d65c1d5dfe82a6655fada53119d841#A#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:05,141 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/02c4bb392fb04a06865bbc2080167608 is 50, key is test_row_0/A:col10/1731808383857/Put/seqid=0 2024-11-17T01:53:05,141 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/93036e4afcea419d94025fef1a21f3f7 is 50, key is test_row_0/B:col10/1731808383857/Put/seqid=0 2024-11-17T01:53:05,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742138_1314 (size=13323) 2024-11-17T01:53:05,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742139_1315 (size=13323) 2024-11-17T01:53:05,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T01:53:05,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:05,161 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-17T01:53:05,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:53:05,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:05,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:53:05,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:05,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:53:05,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:05,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/68309bb10cf3475a85704c7ebd21a4b4 is 50, key is test_row_0/A:col10/1731808383968/Put/seqid=0 2024-11-17T01:53:05,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742140_1316 (size=12301) 2024-11-17T01:53:05,481 INFO [Thread-1073 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1073. 2024-11-17T01:53:05,481 DEBUG [Thread-1073 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:53:05,481 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:05,481 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:05,481 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:05,481 INFO [Thread-1067 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1067. 2024-11-17T01:53:05,481 DEBUG [Thread-1067 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:53:05,481 DEBUG [Thread-1067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:05,482 DEBUG [Thread-1067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:05,482 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:05,482 INFO [Thread-1071 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1071. 2024-11-17T01:53:05,482 DEBUG [Thread-1071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:53:05,482 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:05,482 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:05,482 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:05,484 INFO [Thread-1065 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1065. 2024-11-17T01:53:05,484 DEBUG [Thread-1065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:53:05,484 DEBUG [Thread-1065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:05,484 DEBUG [Thread-1065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:05,484 INFO [Thread-1069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1069. 2024-11-17T01:53:05,484 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:05,484 DEBUG [Thread-1069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:53:05,485 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:05,485 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:05,485 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:05,559 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/93036e4afcea419d94025fef1a21f3f7 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/93036e4afcea419d94025fef1a21f3f7 2024-11-17T01:53:05,562 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/02c4bb392fb04a06865bbc2080167608 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/02c4bb392fb04a06865bbc2080167608 2024-11-17T01:53:05,566 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/B of 36d65c1d5dfe82a6655fada53119d841 into 93036e4afcea419d94025fef1a21f3f7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:05,567 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:53:05,567 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/B, priority=13, startTime=1731808385131; duration=0sec 2024-11-17T01:53:05,567 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:05,567 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:B 2024-11-17T01:53:05,567 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-17T01:53:05,567 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-17T01:53:05,568 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-17T01:53:05,568 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. because compaction request was cancelled 2024-11-17T01:53:05,568 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:C 2024-11-17T01:53:05,568 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36d65c1d5dfe82a6655fada53119d841/A of 36d65c1d5dfe82a6655fada53119d841 into 02c4bb392fb04a06865bbc2080167608(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:05,568 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:53:05,568 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., storeName=36d65c1d5dfe82a6655fada53119d841/A, priority=13, startTime=1731808385131; duration=0sec 2024-11-17T01:53:05,568 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:05,568 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36d65c1d5dfe82a6655fada53119d841:A 2024-11-17T01:53:05,570 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/68309bb10cf3475a85704c7ebd21a4b4 2024-11-17T01:53:05,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/de4535273c4f4972871e94fed4df0323 is 50, key is test_row_0/B:col10/1731808383968/Put/seqid=0 2024-11-17T01:53:05,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742141_1317 (size=12301) 2024-11-17T01:53:05,981 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/de4535273c4f4972871e94fed4df0323 2024-11-17T01:53:05,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/8a6933c58b1346cebe28082570ded8f8 is 50, key is test_row_0/C:col10/1731808383968/Put/seqid=0 2024-11-17T01:53:05,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742142_1318 (size=12301) 2024-11-17T01:53:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T01:53:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:53:06,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. as already flushing 2024-11-17T01:53:06,127 INFO [Thread-1062 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1062. 2024-11-17T01:53:06,127 DEBUG [Thread-1062 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:53:06,127 DEBUG [Thread-1062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:06,128 DEBUG [Thread-1062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:06,128 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:06,136 INFO [Thread-1058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1058. 2024-11-17T01:53:06,136 DEBUG [Thread-1058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:53:06,136 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:06,136 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:06,136 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:06,400 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/8a6933c58b1346cebe28082570ded8f8 2024-11-17T01:53:06,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/68309bb10cf3475a85704c7ebd21a4b4 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/68309bb10cf3475a85704c7ebd21a4b4 2024-11-17T01:53:06,409 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/68309bb10cf3475a85704c7ebd21a4b4, entries=150, sequenceid=437, filesize=12.0 K 2024-11-17T01:53:06,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/de4535273c4f4972871e94fed4df0323 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/de4535273c4f4972871e94fed4df0323 2024-11-17T01:53:06,416 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/de4535273c4f4972871e94fed4df0323, entries=150, sequenceid=437, filesize=12.0 K 2024-11-17T01:53:06,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/8a6933c58b1346cebe28082570ded8f8 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/8a6933c58b1346cebe28082570ded8f8 2024-11-17T01:53:06,421 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/8a6933c58b1346cebe28082570ded8f8, entries=150, sequenceid=437, filesize=12.0 K 2024-11-17T01:53:06,422 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=13.42 KB/13740 for 36d65c1d5dfe82a6655fada53119d841 in 1261ms, sequenceid=437, compaction requested=true 2024-11-17T01:53:06,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 36d65c1d5dfe82a6655fada53119d841: 2024-11-17T01:53:06,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:06,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-17T01:53:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-17T01:53:06,425 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-17T01:53:06,425 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4860 sec 2024-11-17T01:53:06,426 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.4900 sec 2024-11-17T01:53:07,326 INFO [Thread-1054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1054. 2024-11-17T01:53:07,326 DEBUG [Thread-1054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:53:07,326 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:07,326 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:07,326 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:07,357 INFO [Thread-1056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1056. 2024-11-17T01:53:07,357 DEBUG [Thread-1056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:53:07,357 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:07,357 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:07,357 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:07,408 INFO [Thread-1060 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1060. 2024-11-17T01:53:07,408 DEBUG [Thread-1060 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:53:07,408 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:07,408 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:07,409 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:08,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T01:53:08,104 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:08,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T01:53:08,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-17T01:53:08,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-17T01:53:08,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 152 2024-11-17T01:53:08,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-17T01:53:08,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-11-17T01:53:08,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T01:53:08,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2902 2024-11-17T01:53:08,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2848 2024-11-17T01:53:08,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3006 2024-11-17T01:53:08,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2888 2024-11-17T01:53:08,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2852 2024-11-17T01:53:08,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T01:53:08,105 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T01:53:08,105 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:53:08,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:08,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:08,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:08,106 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T01:53:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T01:53:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T01:53:08,109 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808388108"}]},"ts":"1731808388108"} 2024-11-17T01:53:08,110 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T01:53:08,110 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T01:53:08,110 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T01:53:08,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=36d65c1d5dfe82a6655fada53119d841, UNASSIGN}] 2024-11-17T01:53:08,112 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=36d65c1d5dfe82a6655fada53119d841, UNASSIGN 2024-11-17T01:53:08,113 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=36d65c1d5dfe82a6655fada53119d841, regionState=CLOSING, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:08,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=36d65c1d5dfe82a6655fada53119d841, UNASSIGN because future has completed 2024-11-17T01:53:08,114 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T01:53:08,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173}] 2024-11-17T01:53:08,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T01:53:08,268 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:53:08,268 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T01:53:08,268 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 36d65c1d5dfe82a6655fada53119d841, disabling compactions & flushes 2024-11-17T01:53:08,268 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:08,268 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:08,268 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. after waiting 0 ms 2024-11-17T01:53:08,268 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:08,268 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing 36d65c1d5dfe82a6655fada53119d841 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-17T01:53:08,269 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=A 2024-11-17T01:53:08,269 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:08,269 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=B 2024-11-17T01:53:08,269 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:08,269 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36d65c1d5dfe82a6655fada53119d841, store=C 2024-11-17T01:53:08,269 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:08,275 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/061aaa6a76ec424ba47657705376d3d3 is 50, key is test_row_0/A:col10/1731808387355/Put/seqid=0 2024-11-17T01:53:08,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742143_1319 (size=12301) 2024-11-17T01:53:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T01:53:08,679 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/061aaa6a76ec424ba47657705376d3d3 2024-11-17T01:53:08,687 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/45248e86bead42f9b0aaff514470f67c is 50, key is test_row_0/B:col10/1731808387355/Put/seqid=0 2024-11-17T01:53:08,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742144_1320 (size=12301) 2024-11-17T01:53:08,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T01:53:09,094 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/45248e86bead42f9b0aaff514470f67c 2024-11-17T01:53:09,101 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/6365e22f10ec48aab2a2673be6a77414 is 50, key is test_row_0/C:col10/1731808387355/Put/seqid=0 2024-11-17T01:53:09,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742145_1321 (size=12301) 2024-11-17T01:53:09,107 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/6365e22f10ec48aab2a2673be6a77414 2024-11-17T01:53:09,112 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/A/061aaa6a76ec424ba47657705376d3d3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/061aaa6a76ec424ba47657705376d3d3 2024-11-17T01:53:09,116 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/061aaa6a76ec424ba47657705376d3d3, entries=150, sequenceid=447, filesize=12.0 K 2024-11-17T01:53:09,116 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/B/45248e86bead42f9b0aaff514470f67c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/45248e86bead42f9b0aaff514470f67c 2024-11-17T01:53:09,121 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/45248e86bead42f9b0aaff514470f67c, entries=150, sequenceid=447, filesize=12.0 K 2024-11-17T01:53:09,122 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/.tmp/C/6365e22f10ec48aab2a2673be6a77414 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/6365e22f10ec48aab2a2673be6a77414 2024-11-17T01:53:09,129 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/6365e22f10ec48aab2a2673be6a77414, entries=150, sequenceid=447, filesize=12.0 K 2024-11-17T01:53:09,130 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 36d65c1d5dfe82a6655fada53119d841 in 862ms, sequenceid=447, compaction requested=true 2024-11-17T01:53:09,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/312e0d64b11e44b1b4901fe435eafcee, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/736f4aae2b0245da9363906198b7f2d5, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/0b34cf11592a47d89f2eb9793a96ab72, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/71f59d1044684ca4b30a57277dcaecc1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/735702213e704c16ba6ea428958b3a5d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/11876b0ad9104e9a8de810af17707933, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/eddc848a981f4ae7a2e08abf9511b181, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/1c6c31d5243640c689c51331f3f5ba43, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/d7eb353de3514ddea32ac03e440950ee, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/5343d305cb164ef3ae3d7a33dcb7e163, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/b8a09ddf78484f32b952224a65371bc3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/e2d76511f4254e598657c85525f05072, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/ca2d323d23074b18b67dc858742514a5, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/6de3fde55d5849fabdbfb0c7e6f0e8d3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/ab62d5f28ddd4a368201792e279deb64, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/9f2361a57571476ea2ddd45ed79c4277, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/2e30136929374096b1bb357bbc3d6094, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/7e81d40cae9d4e7a899ccd438f71969e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/f192449fbd30467f8da19b6576180ebd, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/67464f77a6034e4e93136a4c36fc6a90, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/fc6fe5f49e0349828ef4c15ad7a83833, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/1dd23857dd394cb9bd32c0cfcd84b9d7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/cda097e985154fd19983362e1fd06ced, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/e9837b9c26ca44228472a7ca643ee94b, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/b4d9c576e3c7496594ddf9028e23847e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/c014e15ccc9745e8b7419fdf40ba5e8c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/1ed9105120914068bc77d40c6cf1c8a5, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/6beee640a5424935834c35f9f160d377, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/b3d1a7fe459b4ab191171bef5922e725, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/bfcee8d817bc4aac996de01241779226] to archive 2024-11-17T01:53:09,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T01:53:09,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/312e0d64b11e44b1b4901fe435eafcee to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/312e0d64b11e44b1b4901fe435eafcee 2024-11-17T01:53:09,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/736f4aae2b0245da9363906198b7f2d5 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/736f4aae2b0245da9363906198b7f2d5 2024-11-17T01:53:09,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/0b34cf11592a47d89f2eb9793a96ab72 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/0b34cf11592a47d89f2eb9793a96ab72 2024-11-17T01:53:09,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/71f59d1044684ca4b30a57277dcaecc1 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/71f59d1044684ca4b30a57277dcaecc1 2024-11-17T01:53:09,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/735702213e704c16ba6ea428958b3a5d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/735702213e704c16ba6ea428958b3a5d 2024-11-17T01:53:09,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/11876b0ad9104e9a8de810af17707933 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/11876b0ad9104e9a8de810af17707933 2024-11-17T01:53:09,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/eddc848a981f4ae7a2e08abf9511b181 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/eddc848a981f4ae7a2e08abf9511b181 2024-11-17T01:53:09,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/1c6c31d5243640c689c51331f3f5ba43 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/1c6c31d5243640c689c51331f3f5ba43 2024-11-17T01:53:09,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/d7eb353de3514ddea32ac03e440950ee to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/d7eb353de3514ddea32ac03e440950ee 2024-11-17T01:53:09,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/5343d305cb164ef3ae3d7a33dcb7e163 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/5343d305cb164ef3ae3d7a33dcb7e163 2024-11-17T01:53:09,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/b8a09ddf78484f32b952224a65371bc3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/b8a09ddf78484f32b952224a65371bc3 2024-11-17T01:53:09,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/e2d76511f4254e598657c85525f05072 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/e2d76511f4254e598657c85525f05072 2024-11-17T01:53:09,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/ca2d323d23074b18b67dc858742514a5 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/ca2d323d23074b18b67dc858742514a5 2024-11-17T01:53:09,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/6de3fde55d5849fabdbfb0c7e6f0e8d3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/6de3fde55d5849fabdbfb0c7e6f0e8d3 2024-11-17T01:53:09,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/ab62d5f28ddd4a368201792e279deb64 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/ab62d5f28ddd4a368201792e279deb64 2024-11-17T01:53:09,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/9f2361a57571476ea2ddd45ed79c4277 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/9f2361a57571476ea2ddd45ed79c4277 2024-11-17T01:53:09,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/2e30136929374096b1bb357bbc3d6094 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/2e30136929374096b1bb357bbc3d6094 2024-11-17T01:53:09,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/7e81d40cae9d4e7a899ccd438f71969e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/7e81d40cae9d4e7a899ccd438f71969e 2024-11-17T01:53:09,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/f192449fbd30467f8da19b6576180ebd to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/f192449fbd30467f8da19b6576180ebd 2024-11-17T01:53:09,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/67464f77a6034e4e93136a4c36fc6a90 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/67464f77a6034e4e93136a4c36fc6a90 2024-11-17T01:53:09,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/fc6fe5f49e0349828ef4c15ad7a83833 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/fc6fe5f49e0349828ef4c15ad7a83833 2024-11-17T01:53:09,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/1dd23857dd394cb9bd32c0cfcd84b9d7 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/1dd23857dd394cb9bd32c0cfcd84b9d7 2024-11-17T01:53:09,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/cda097e985154fd19983362e1fd06ced to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/cda097e985154fd19983362e1fd06ced 2024-11-17T01:53:09,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/e9837b9c26ca44228472a7ca643ee94b to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/e9837b9c26ca44228472a7ca643ee94b 2024-11-17T01:53:09,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/b4d9c576e3c7496594ddf9028e23847e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/b4d9c576e3c7496594ddf9028e23847e 2024-11-17T01:53:09,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/c014e15ccc9745e8b7419fdf40ba5e8c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/c014e15ccc9745e8b7419fdf40ba5e8c 2024-11-17T01:53:09,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/1ed9105120914068bc77d40c6cf1c8a5 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/1ed9105120914068bc77d40c6cf1c8a5 2024-11-17T01:53:09,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/6beee640a5424935834c35f9f160d377 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/6beee640a5424935834c35f9f160d377 2024-11-17T01:53:09,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/b3d1a7fe459b4ab191171bef5922e725 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/b3d1a7fe459b4ab191171bef5922e725 2024-11-17T01:53:09,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/bfcee8d817bc4aac996de01241779226 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/bfcee8d817bc4aac996de01241779226 2024-11-17T01:53:09,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/20dc0285c3b7433bba05c4cd095ff9b0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/47c3b45deb29496693a78d89ca9b1f39, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/63cd5dc383a341159d3f0e3b24784296, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/b13e01c4d56c44528f1506c265f04e58, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/cbe9ec48709b4c6083a1d968cfdfa36d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/0319ffeb41f84c3e9d78ddc5f2346d15, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/83135d38d0f343b48def8fceb449812b, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/bc96d0d4551649539b2569fcdd420abd, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/ed8eaeeb10844fe38983aa3597e94376, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/cef21c9b950d4886ba8f297e7ae3452e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/67bc8661f6684410a304834847c05d8f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/98f0bc29bb8d4151ac20d45a9c28c723, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/a0dd3f5c3d5449ee867fb57e4e1bf142, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/cc252ffbe7c047c48cc53c434294144d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/45c095eec5fe46ef92c967b12586de8a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/71efd0893b034109ae865e076ef4f5d4, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/a2e7232a0aee4af396873f7eba114813, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/282662365084434599834077d64335cd, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/c372a41148ca451eb67634ef5810c293, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/a22ccdfc4d1f4b93958fdc16400b8449, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/f9734ba6e0f24403ae8e175249f1ef6f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/31e72589c2c5403196ec836e256551ef, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/7145fbf134ce4ca183b1dd2044a8b7b5, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/b81607b7ba07445fad723a0ef610c3c2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/0f94821b96d643a5a68b8530ef7cddec, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/33b22d9fb2044ef2a604346f69bbe723, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/b188a83ad8d1452f91399068dd4d8dc3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/02f3cf92beae4d11b5e270ea6c0ee83d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/2aa5ea551dfe4873a3d7045de040e1f3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/52b7c62f3de947e28c45c177f1f08304] to archive 2024-11-17T01:53:09,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T01:53:09,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/20dc0285c3b7433bba05c4cd095ff9b0 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/20dc0285c3b7433bba05c4cd095ff9b0 2024-11-17T01:53:09,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/47c3b45deb29496693a78d89ca9b1f39 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/47c3b45deb29496693a78d89ca9b1f39 2024-11-17T01:53:09,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/63cd5dc383a341159d3f0e3b24784296 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/63cd5dc383a341159d3f0e3b24784296 2024-11-17T01:53:09,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/b13e01c4d56c44528f1506c265f04e58 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/b13e01c4d56c44528f1506c265f04e58 2024-11-17T01:53:09,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/cbe9ec48709b4c6083a1d968cfdfa36d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/cbe9ec48709b4c6083a1d968cfdfa36d 2024-11-17T01:53:09,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/0319ffeb41f84c3e9d78ddc5f2346d15 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/0319ffeb41f84c3e9d78ddc5f2346d15 2024-11-17T01:53:09,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/83135d38d0f343b48def8fceb449812b to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/83135d38d0f343b48def8fceb449812b 2024-11-17T01:53:09,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/bc96d0d4551649539b2569fcdd420abd to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/bc96d0d4551649539b2569fcdd420abd 2024-11-17T01:53:09,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/ed8eaeeb10844fe38983aa3597e94376 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/ed8eaeeb10844fe38983aa3597e94376 2024-11-17T01:53:09,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/cef21c9b950d4886ba8f297e7ae3452e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/cef21c9b950d4886ba8f297e7ae3452e 2024-11-17T01:53:09,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/67bc8661f6684410a304834847c05d8f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/67bc8661f6684410a304834847c05d8f 2024-11-17T01:53:09,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/98f0bc29bb8d4151ac20d45a9c28c723 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/98f0bc29bb8d4151ac20d45a9c28c723 2024-11-17T01:53:09,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/a0dd3f5c3d5449ee867fb57e4e1bf142 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/a0dd3f5c3d5449ee867fb57e4e1bf142 2024-11-17T01:53:09,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/cc252ffbe7c047c48cc53c434294144d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/cc252ffbe7c047c48cc53c434294144d 2024-11-17T01:53:09,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/45c095eec5fe46ef92c967b12586de8a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/45c095eec5fe46ef92c967b12586de8a 2024-11-17T01:53:09,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/71efd0893b034109ae865e076ef4f5d4 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/71efd0893b034109ae865e076ef4f5d4 2024-11-17T01:53:09,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/a2e7232a0aee4af396873f7eba114813 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/a2e7232a0aee4af396873f7eba114813 2024-11-17T01:53:09,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/282662365084434599834077d64335cd to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/282662365084434599834077d64335cd 2024-11-17T01:53:09,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/c372a41148ca451eb67634ef5810c293 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/c372a41148ca451eb67634ef5810c293 2024-11-17T01:53:09,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/a22ccdfc4d1f4b93958fdc16400b8449 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/a22ccdfc4d1f4b93958fdc16400b8449 2024-11-17T01:53:09,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/f9734ba6e0f24403ae8e175249f1ef6f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/f9734ba6e0f24403ae8e175249f1ef6f 2024-11-17T01:53:09,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/31e72589c2c5403196ec836e256551ef to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/31e72589c2c5403196ec836e256551ef 2024-11-17T01:53:09,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/7145fbf134ce4ca183b1dd2044a8b7b5 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/7145fbf134ce4ca183b1dd2044a8b7b5 2024-11-17T01:53:09,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/b81607b7ba07445fad723a0ef610c3c2 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/b81607b7ba07445fad723a0ef610c3c2 2024-11-17T01:53:09,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/0f94821b96d643a5a68b8530ef7cddec to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/0f94821b96d643a5a68b8530ef7cddec 2024-11-17T01:53:09,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/33b22d9fb2044ef2a604346f69bbe723 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/33b22d9fb2044ef2a604346f69bbe723 2024-11-17T01:53:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T01:53:09,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/b188a83ad8d1452f91399068dd4d8dc3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/b188a83ad8d1452f91399068dd4d8dc3 2024-11-17T01:53:09,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/02f3cf92beae4d11b5e270ea6c0ee83d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/02f3cf92beae4d11b5e270ea6c0ee83d 2024-11-17T01:53:09,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/2aa5ea551dfe4873a3d7045de040e1f3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/2aa5ea551dfe4873a3d7045de040e1f3 2024-11-17T01:53:09,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/52b7c62f3de947e28c45c177f1f08304 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/52b7c62f3de947e28c45c177f1f08304 2024-11-17T01:53:09,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/377a8b050eae47ee9bd1aef069d7232b, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/99338179099442b2bb829bab34d86d06, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/445cc0befa8b471fb3a55b8b7ca1791c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/1a5ebf8fcf484f8d9d79df5c72af16f6, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/9e86b203407545edbca37bc5b8903f78, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/64c14c63e22b488396acfbd99f1e8f52, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/ac4a2bdca37f4220870917c5c443cb63, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/d11f78a09f5244e4875d330dc40ba5cd, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/270e11e9b475434ca4138ed5e042b26c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/15bee743d3ef4105a112a6cd861f10dc, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/f6310d656beb4be2bc390264c5204f2a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/4f31da9d504244d7afca131db420bdd8, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/f21ce9a578b64533b86d001c3cd70398, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e07a4767716a41e49aed46909fba1158, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/2fc0673aad6444cc84c21a7c13c18de3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e510773d41644e29b2b27f81a5d60935, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/24aae12665e440c69dfef5552b6073fa, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/44ea4d98b2014db283fa961546e9114c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/7fa14a41c22c485f92f5c72001d19006, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/d3bf1c3ad74641a180dadb68b730ad6b, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/283cdc9a5946477198a3a522b73721f7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e26f044cd6bb4b4ba3358c5e646f333f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/d2b882f8fdad4e8a9b1723523922dcb5, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/51cd98bdcd4846a58acb2efdd5701d49, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/ad46cf7476fd4a5db369eae429f8151d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/948fd5115bab4ca8a1817157b7209d07, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/c2486df61c014c5a8bd922fd61a64c3e] to archive 2024-11-17T01:53:09,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T01:53:09,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/377a8b050eae47ee9bd1aef069d7232b to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/377a8b050eae47ee9bd1aef069d7232b 2024-11-17T01:53:09,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/99338179099442b2bb829bab34d86d06 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/99338179099442b2bb829bab34d86d06 2024-11-17T01:53:09,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/445cc0befa8b471fb3a55b8b7ca1791c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/445cc0befa8b471fb3a55b8b7ca1791c 2024-11-17T01:53:09,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/1a5ebf8fcf484f8d9d79df5c72af16f6 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/1a5ebf8fcf484f8d9d79df5c72af16f6 2024-11-17T01:53:09,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/9e86b203407545edbca37bc5b8903f78 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/9e86b203407545edbca37bc5b8903f78 2024-11-17T01:53:09,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/64c14c63e22b488396acfbd99f1e8f52 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/64c14c63e22b488396acfbd99f1e8f52 2024-11-17T01:53:09,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/ac4a2bdca37f4220870917c5c443cb63 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/ac4a2bdca37f4220870917c5c443cb63 2024-11-17T01:53:09,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/d11f78a09f5244e4875d330dc40ba5cd to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/d11f78a09f5244e4875d330dc40ba5cd 2024-11-17T01:53:09,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/270e11e9b475434ca4138ed5e042b26c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/270e11e9b475434ca4138ed5e042b26c 2024-11-17T01:53:09,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/15bee743d3ef4105a112a6cd861f10dc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/15bee743d3ef4105a112a6cd861f10dc 2024-11-17T01:53:09,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/f6310d656beb4be2bc390264c5204f2a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/f6310d656beb4be2bc390264c5204f2a 2024-11-17T01:53:09,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/4f31da9d504244d7afca131db420bdd8 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/4f31da9d504244d7afca131db420bdd8 2024-11-17T01:53:09,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/f21ce9a578b64533b86d001c3cd70398 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/f21ce9a578b64533b86d001c3cd70398 2024-11-17T01:53:09,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e07a4767716a41e49aed46909fba1158 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e07a4767716a41e49aed46909fba1158 2024-11-17T01:53:09,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/2fc0673aad6444cc84c21a7c13c18de3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/2fc0673aad6444cc84c21a7c13c18de3 2024-11-17T01:53:09,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e510773d41644e29b2b27f81a5d60935 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e510773d41644e29b2b27f81a5d60935 2024-11-17T01:53:09,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/24aae12665e440c69dfef5552b6073fa to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/24aae12665e440c69dfef5552b6073fa 2024-11-17T01:53:09,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/44ea4d98b2014db283fa961546e9114c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/44ea4d98b2014db283fa961546e9114c 2024-11-17T01:53:09,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/7fa14a41c22c485f92f5c72001d19006 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/7fa14a41c22c485f92f5c72001d19006 2024-11-17T01:53:09,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/d3bf1c3ad74641a180dadb68b730ad6b to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/d3bf1c3ad74641a180dadb68b730ad6b 2024-11-17T01:53:09,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/283cdc9a5946477198a3a522b73721f7 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/283cdc9a5946477198a3a522b73721f7 2024-11-17T01:53:09,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e26f044cd6bb4b4ba3358c5e646f333f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e26f044cd6bb4b4ba3358c5e646f333f 2024-11-17T01:53:09,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/d2b882f8fdad4e8a9b1723523922dcb5 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/d2b882f8fdad4e8a9b1723523922dcb5 2024-11-17T01:53:09,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/51cd98bdcd4846a58acb2efdd5701d49 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/51cd98bdcd4846a58acb2efdd5701d49 2024-11-17T01:53:09,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/ad46cf7476fd4a5db369eae429f8151d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/ad46cf7476fd4a5db369eae429f8151d 2024-11-17T01:53:09,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/948fd5115bab4ca8a1817157b7209d07 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/948fd5115bab4ca8a1817157b7209d07 2024-11-17T01:53:09,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/c2486df61c014c5a8bd922fd61a64c3e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/c2486df61c014c5a8bd922fd61a64c3e 2024-11-17T01:53:09,313 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/recovered.edits/450.seqid, newMaxSeqId=450, maxSeqId=1 2024-11-17T01:53:09,314 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. 2024-11-17T01:53:09,314 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 36d65c1d5dfe82a6655fada53119d841: Waiting for close lock at 1731808388268Running coprocessor pre-close hooks at 1731808388268Disabling compacts and flushes for region at 1731808388268Disabling writes for close at 1731808388268Obtaining lock to block concurrent updates at 1731808388268Preparing flush snapshotting stores in 36d65c1d5dfe82a6655fada53119d841 at 1731808388268Finished memstore snapshotting TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731808388269 (+1 ms)Flushing stores of TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841. at 1731808388270 (+1 ms)Flushing 36d65c1d5dfe82a6655fada53119d841/A: creating writer at 1731808388270Flushing 36d65c1d5dfe82a6655fada53119d841/A: appending metadata at 1731808388275 (+5 ms)Flushing 36d65c1d5dfe82a6655fada53119d841/A: closing flushed file at 1731808388275Flushing 36d65c1d5dfe82a6655fada53119d841/B: creating writer at 1731808388684 (+409 ms)Flushing 36d65c1d5dfe82a6655fada53119d841/B: appending metadata at 1731808388687 (+3 ms)Flushing 36d65c1d5dfe82a6655fada53119d841/B: closing flushed file at 1731808388687Flushing 36d65c1d5dfe82a6655fada53119d841/C: creating writer at 1731808389097 (+410 ms)Flushing 36d65c1d5dfe82a6655fada53119d841/C: appending metadata at 1731808389100 (+3 ms)Flushing 36d65c1d5dfe82a6655fada53119d841/C: closing flushed file at 1731808389100Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56ca4fac: reopening flushed file at 1731808389111 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d59074e: reopening flushed file at 1731808389116 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24cb39d0: reopening flushed file at 1731808389121 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 36d65c1d5dfe82a6655fada53119d841 in 862ms, sequenceid=447, compaction requested=true at 1731808389130 (+9 ms)Writing region close event to WAL at 1731808389310 (+180 ms)Running coprocessor post-close hooks at 1731808389314 (+4 ms)Closed at 1731808389314 2024-11-17T01:53:09,316 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:53:09,316 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=36d65c1d5dfe82a6655fada53119d841, regionState=CLOSED 2024-11-17T01:53:09,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 because future has completed 2024-11-17T01:53:09,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-17T01:53:09,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 36d65c1d5dfe82a6655fada53119d841, server=c6c0d7a0a7c0,41157,1731808292173 in 1.2040 sec 2024-11-17T01:53:09,322 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-17T01:53:09,322 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=36d65c1d5dfe82a6655fada53119d841, UNASSIGN in 1.2080 sec 2024-11-17T01:53:09,324 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-17T01:53:09,324 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.2130 sec 2024-11-17T01:53:09,326 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808389325"}]},"ts":"1731808389325"} 2024-11-17T01:53:09,327 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T01:53:09,327 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T01:53:09,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.2230 sec 2024-11-17T01:53:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T01:53:10,254 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:10,255 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T01:53:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:53:10,257 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:53:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T01:53:10,258 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:53:10,259 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:53:10,262 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A, FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B, FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C, FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/recovered.edits] 2024-11-17T01:53:10,264 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/02c4bb392fb04a06865bbc2080167608 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/02c4bb392fb04a06865bbc2080167608 2024-11-17T01:53:10,266 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/061aaa6a76ec424ba47657705376d3d3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/061aaa6a76ec424ba47657705376d3d3 2024-11-17T01:53:10,267 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/68309bb10cf3475a85704c7ebd21a4b4 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/A/68309bb10cf3475a85704c7ebd21a4b4 2024-11-17T01:53:10,271 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/45248e86bead42f9b0aaff514470f67c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/45248e86bead42f9b0aaff514470f67c 2024-11-17T01:53:10,273 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/93036e4afcea419d94025fef1a21f3f7 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/93036e4afcea419d94025fef1a21f3f7 2024-11-17T01:53:10,275 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/de4535273c4f4972871e94fed4df0323 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/B/de4535273c4f4972871e94fed4df0323 2024-11-17T01:53:10,277 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/6365e22f10ec48aab2a2673be6a77414 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/6365e22f10ec48aab2a2673be6a77414 2024-11-17T01:53:10,279 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/8a6933c58b1346cebe28082570ded8f8 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/8a6933c58b1346cebe28082570ded8f8 2024-11-17T01:53:10,280 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/dafb90eea8784ff6a84a0665065d1dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/dafb90eea8784ff6a84a0665065d1dcc 2024-11-17T01:53:10,282 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e91dad38226048ada10e9ecc38b00cc0 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/C/e91dad38226048ada10e9ecc38b00cc0 2024-11-17T01:53:10,286 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/recovered.edits/450.seqid to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841/recovered.edits/450.seqid 2024-11-17T01:53:10,286 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/36d65c1d5dfe82a6655fada53119d841 2024-11-17T01:53:10,286 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T01:53:10,289 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:53:10,292 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T01:53:10,295 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T01:53:10,296 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:53:10,296 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T01:53:10,296 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731808390296"}]},"ts":"9223372036854775807"} 2024-11-17T01:53:10,298 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T01:53:10,298 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 36d65c1d5dfe82a6655fada53119d841, NAME => 'TestAcidGuarantees,,1731808363279.36d65c1d5dfe82a6655fada53119d841.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T01:53:10,298 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T01:53:10,299 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731808390298"}]},"ts":"9223372036854775807"} 2024-11-17T01:53:10,301 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T01:53:10,302 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:53:10,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-11-17T01:53:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T01:53:10,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T01:53:10,364 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:10,373 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_940601892_22 at /127.0.0.1:46556 [Waiting for operation #366] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1864014360_22 at /127.0.0.1:46774 [Waiting for operation #237] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=489 (was 453) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4857 (was 5022) 2024-11-17T01:53:10,383 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=489, ProcessCount=11, AvailableMemoryMB=4856 2024-11-17T01:53:10,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T01:53:10,385 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:53:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T01:53:10,387 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T01:53:10,387 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:10,387 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-17T01:53:10,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-17T01:53:10,388 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T01:53:10,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742146_1322 (size=1061) 2024-11-17T01:53:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-17T01:53:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-17T01:53:10,795 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1 2024-11-17T01:53:10,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742147_1323 (size=53) 2024-11-17T01:53:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-17T01:53:11,204 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:53:11,204 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fe092165677ecd92ff68c61a7627feb9, disabling compactions & flushes 2024-11-17T01:53:11,204 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:11,204 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:11,204 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. after waiting 0 ms 2024-11-17T01:53:11,204 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:11,204 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:11,204 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fe092165677ecd92ff68c61a7627feb9: Waiting for close lock at 1731808391204Disabling compacts and flushes for region at 1731808391204Disabling writes for close at 1731808391204Writing region close event to WAL at 1731808391204Closed at 1731808391204 2024-11-17T01:53:11,205 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T01:53:11,206 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731808391205"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731808391205"}]},"ts":"1731808391205"} 2024-11-17T01:53:11,207 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T01:53:11,208 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T01:53:11,208 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808391208"}]},"ts":"1731808391208"} 2024-11-17T01:53:11,210 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T01:53:11,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe092165677ecd92ff68c61a7627feb9, ASSIGN}] 2024-11-17T01:53:11,211 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe092165677ecd92ff68c61a7627feb9, ASSIGN 2024-11-17T01:53:11,212 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe092165677ecd92ff68c61a7627feb9, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,41157,1731808292173; forceNewPlan=false, retain=false 2024-11-17T01:53:11,362 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=fe092165677ecd92ff68c61a7627feb9, regionState=OPENING, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:11,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe092165677ecd92ff68c61a7627feb9, ASSIGN because future has completed 2024-11-17T01:53:11,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173}] 2024-11-17T01:53:11,518 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:11,519 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:53:11,519 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:11,519 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:53:11,519 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:11,519 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:11,520 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:11,521 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:53:11,521 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe092165677ecd92ff68c61a7627feb9 columnFamilyName A 2024-11-17T01:53:11,521 DEBUG [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:11,522 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] regionserver.HStore(327): Store=fe092165677ecd92ff68c61a7627feb9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:53:11,522 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:11,523 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:53:11,523 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe092165677ecd92ff68c61a7627feb9 columnFamilyName B 2024-11-17T01:53:11,523 DEBUG [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:11,523 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] regionserver.HStore(327): Store=fe092165677ecd92ff68c61a7627feb9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:53:11,523 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:11,524 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:53:11,524 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe092165677ecd92ff68c61a7627feb9 columnFamilyName C 2024-11-17T01:53:11,524 DEBUG [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-17T01:53:11,524 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] regionserver.HStore(327): Store=fe092165677ecd92ff68c61a7627feb9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:53:11,524 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:11,524 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:11,525 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:11,525 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:11,526 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:11,526 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:11,526 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T01:53:11,527 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:11,528 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:53:11,529 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened fe092165677ecd92ff68c61a7627feb9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70125164, jitterRate=0.044946372509002686}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T01:53:11,529 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:11,529 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for fe092165677ecd92ff68c61a7627feb9: Running coprocessor pre-open hook at 1731808391519Writing region info on filesystem at 1731808391519Initializing all the Stores at 1731808391520 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808391520Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808391520Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808391520Cleaning up temporary data from old regions at 1731808391526 (+6 ms)Running coprocessor post-open hooks at 1731808391529 (+3 ms)Region opened successfully at 1731808391529 2024-11-17T01:53:11,530 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., pid=91, masterSystemTime=1731808391516 2024-11-17T01:53:11,531 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:11,531 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:11,532 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=fe092165677ecd92ff68c61a7627feb9, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:11,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 because future has completed 2024-11-17T01:53:11,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-17T01:53:11,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 in 169 msec 2024-11-17T01:53:11,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-17T01:53:11,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe092165677ecd92ff68c61a7627feb9, ASSIGN in 325 msec 2024-11-17T01:53:11,537 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T01:53:11,537 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808391537"}]},"ts":"1731808391537"} 2024-11-17T01:53:11,538 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T01:53:11,539 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T01:53:11,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1540 sec 2024-11-17T01:53:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-17T01:53:12,534 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:12,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300a5f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:12,535 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:12,536 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:12,537 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:12,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:12,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:12,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c163a0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:12,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:12,539 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ee338e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:12,540 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:12,542 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:12,543 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T01:53:12,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c6c0d7a0a7c0,45023,1731808291372 2024-11-17T01:53:12,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@36c1cc2e 2024-11-17T01:53:12,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T01:53:12,544 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T01:53:12,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T01:53:12,546 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:53:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T01:53:12,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742148_1324 (size=1097) 2024-11-17T01:53:12,963 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-17T01:53:12,963 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-17T01:53:12,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T01:53:12,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe092165677ecd92ff68c61a7627feb9, REOPEN/MOVE}] 2024-11-17T01:53:12,970 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe092165677ecd92ff68c61a7627feb9, REOPEN/MOVE 2024-11-17T01:53:12,971 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=fe092165677ecd92ff68c61a7627feb9, regionState=CLOSING, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:12,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe092165677ecd92ff68c61a7627feb9, REOPEN/MOVE because future has completed 2024-11-17T01:53:12,972 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T01:53:12,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173}] 2024-11-17T01:53:13,125 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:13,125 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T01:53:13,125 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing fe092165677ecd92ff68c61a7627feb9, disabling compactions & flushes 2024-11-17T01:53:13,125 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:13,125 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:13,125 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. after waiting 0 ms 2024-11-17T01:53:13,125 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:13,129 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T01:53:13,130 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:13,130 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for fe092165677ecd92ff68c61a7627feb9: Waiting for close lock at 1731808393125Running coprocessor pre-close hooks at 1731808393125Disabling compacts and flushes for region at 1731808393125Disabling writes for close at 1731808393125Writing region close event to WAL at 1731808393126 (+1 ms)Running coprocessor post-close hooks at 1731808393130 (+4 ms)Closed at 1731808393130 2024-11-17T01:53:13,130 WARN [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: fe092165677ecd92ff68c61a7627feb9 to self. 2024-11-17T01:53:13,131 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:13,132 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=fe092165677ecd92ff68c61a7627feb9, regionState=CLOSED 2024-11-17T01:53:13,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 because future has completed 2024-11-17T01:53:13,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-17T01:53:13,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 in 162 msec 2024-11-17T01:53:13,136 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe092165677ecd92ff68c61a7627feb9, REOPEN/MOVE; state=CLOSED, location=c6c0d7a0a7c0,41157,1731808292173; forceNewPlan=false, retain=true 2024-11-17T01:53:13,287 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=fe092165677ecd92ff68c61a7627feb9, regionState=OPENING, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:13,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe092165677ecd92ff68c61a7627feb9, REOPEN/MOVE because future has completed 2024-11-17T01:53:13,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173}] 2024-11-17T01:53:13,444 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:13,444 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:53:13,444 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:13,444 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:53:13,445 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:13,445 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:13,446 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:13,447 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:53:13,447 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe092165677ecd92ff68c61a7627feb9 columnFamilyName A 2024-11-17T01:53:13,448 DEBUG [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:13,448 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] regionserver.HStore(327): Store=fe092165677ecd92ff68c61a7627feb9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:53:13,449 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:13,450 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:53:13,450 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe092165677ecd92ff68c61a7627feb9 columnFamilyName B 2024-11-17T01:53:13,450 DEBUG [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:13,451 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] regionserver.HStore(327): Store=fe092165677ecd92ff68c61a7627feb9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:53:13,451 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:13,451 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:53:13,452 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe092165677ecd92ff68c61a7627feb9 columnFamilyName C 2024-11-17T01:53:13,452 DEBUG [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:13,452 INFO [StoreOpener-fe092165677ecd92ff68c61a7627feb9-1 {}] regionserver.HStore(327): Store=fe092165677ecd92ff68c61a7627feb9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:53:13,452 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:13,452 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:13,453 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:13,454 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:13,455 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:13,455 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:13,455 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T01:53:13,457 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:13,458 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened fe092165677ecd92ff68c61a7627feb9; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67991618, jitterRate=0.013154059648513794}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T01:53:13,458 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:13,459 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for fe092165677ecd92ff68c61a7627feb9: Running coprocessor pre-open hook at 1731808393445Writing region info on filesystem at 1731808393445Initializing all the Stores at 1731808393445Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808393445Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808393446 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808393446Cleaning up temporary data from old regions at 1731808393455 (+9 ms)Running coprocessor post-open hooks at 1731808393458 (+3 ms)Region opened successfully at 1731808393459 (+1 ms) 2024-11-17T01:53:13,460 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., pid=96, masterSystemTime=1731808393441 2024-11-17T01:53:13,462 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:13,462 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:13,462 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=fe092165677ecd92ff68c61a7627feb9, regionState=OPEN, openSeqNum=5, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:13,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 because future has completed 2024-11-17T01:53:13,467 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-17T01:53:13,467 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 in 176 msec 2024-11-17T01:53:13,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-17T01:53:13,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe092165677ecd92ff68c61a7627feb9, REOPEN/MOVE in 498 msec 2024-11-17T01:53:13,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-17T01:53:13,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 504 msec 2024-11-17T01:53:13,474 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 926 msec 2024-11-17T01:53:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-17T01:53:13,475 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:13,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1774ef6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,476 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:13,476 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:13,477 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:13,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f3a211, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:13,479 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc6bb1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,480 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,480 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:13,480 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:13,481 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c4f9191, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:13,482 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:13,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b6e98a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,483 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:13,483 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:13,484 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:13,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:13,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ee19e87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:13,486 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:13,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28a84b37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,487 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,487 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:13,487 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:13,488 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:13,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:13,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:13,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@417b4d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:13,489 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4021fc8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,491 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:13,491 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:13,492 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@624867ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:13,493 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae6ad20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,494 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:13,494 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:13,495 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e880b24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:13,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:13,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:13,497 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:13,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7490dd1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,498 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:13,498 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:13,499 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a5ed659, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:13,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:13,500 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@563a6be7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,502 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:13,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:13,502 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64985aef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:13,504 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76329fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,505 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:13,506 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:13,506 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:13,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:13,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:13,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff7d5af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:13,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:13,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:13,508 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e8b0ffb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,510 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:13,510 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:13,511 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:13,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:13,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:13,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a8b7dae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:13,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:13,512 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@221b0a03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:13,513 DEBUG [Thread-1490 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:13,513 DEBUG [Thread-1501 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:13,514 DEBUG [Thread-1495 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:13,514 DEBUG [Thread-1493 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:13,515 DEBUG [Thread-1499 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:13,516 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:13,517 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:13,517 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:13,517 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:13,517 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54344, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:53:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:53:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:53:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:53:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:53:13,531 DEBUG [Thread-1482 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:13,534 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:13,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:53:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:13,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T01:53:13,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:13,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:13,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:13,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:13,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:13,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:13,544 DEBUG [Thread-1497 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:13,551 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:13,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:53:13,555 DEBUG [Thread-1488 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:13,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:13,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:53:13,563 DEBUG [Thread-1484 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:13,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:13,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:13,565 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:53:13,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-17T01:53:13,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:53:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T01:53:13,568 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:53:13,570 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:53:13,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:53:13,576 DEBUG [Thread-1486 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:13,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:13,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:13,579 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:13,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f3492866dd384e2db1ae0e5c129f81bd_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_0/A:col10/1731808393533/Put/seqid=0 2024-11-17T01:53:13,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:53:13,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742149_1325 (size=14594) 2024-11-17T01:53:13,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:13,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808453585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:13,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808393582, max timestamp=1731808393596Num uniques 859; 2024-11-17T01:53:13,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808393582, max timestamp=1731808393596Num uniques 859; 2024-11-17T01:53:13,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:53:13,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808393582, max timestamp=1731808393596Num uniques 859; 2024-11-17T01:53:13,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:53:13,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:53:13,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808453650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:13,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808453658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:13,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T01:53:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808453661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:13,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808453663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:13,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808453704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:13,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-17T01:53:13,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:13,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:13,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:13,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:13,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:13,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:13,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:13,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808453774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:13,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808453774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:13,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:13,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808453784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:13,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:13,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:13,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808453784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:13,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:13,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-17T01:53:13,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:13,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:13,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:13,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:13,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:13,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T01:53:13,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:13,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808453915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:13,996 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:14,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:14,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808453994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:14,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808453994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:14,008 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f3492866dd384e2db1ae0e5c129f81bd_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f3492866dd384e2db1ae0e5c129f81bd_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:14,009 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/121c689a7cad40e19c60664c8ff29e14, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:14,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/121c689a7cad40e19c60664c8ff29e14 is 175, key is test_row_0/A:col10/1731808393533/Put/seqid=0 2024-11-17T01:53:14,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:14,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808454006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:14,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808454006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:14,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:14,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742150_1326 (size=39549) 2024-11-17T01:53:14,021 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/121c689a7cad40e19c60664c8ff29e14 2024-11-17T01:53:14,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-17T01:53:14,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:14,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:14,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:14,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:14,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:14,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:14,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/e4f4ed3caca2494dac8355798c52bf92 is 50, key is test_row_0/B:col10/1731808393533/Put/seqid=0 2024-11-17T01:53:14,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742151_1327 (size=12001) 2024-11-17T01:53:14,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-17T01:53:14,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:14,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:14,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:14,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:14,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T01:53:14,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808454224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:14,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808454314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:14,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808454314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:14,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:14,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808454324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:14,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808454324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:14,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-17T01:53:14,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:14,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:14,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:14,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:14,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:14,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:14,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/e4f4ed3caca2494dac8355798c52bf92 2024-11-17T01:53:14,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-17T01:53:14,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:14,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:14,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:14,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:14,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:14,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/ca84c2a725994c4a9517c200f98d9ed0 is 50, key is test_row_0/C:col10/1731808393533/Put/seqid=0 2024-11-17T01:53:14,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742152_1328 (size=12001) 2024-11-17T01:53:14,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-17T01:53:14,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:14,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:14,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:14,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:14,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T01:53:14,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808454735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:14,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-17T01:53:14,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:14,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:14,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:14,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:14,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:14,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:14,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808454834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:14,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808454834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:14,846 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T01:53:14,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808454845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:14,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:14,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808454845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:14,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/ca84c2a725994c4a9517c200f98d9ed0 2024-11-17T01:53:14,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/121c689a7cad40e19c60664c8ff29e14 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/121c689a7cad40e19c60664c8ff29e14 2024-11-17T01:53:14,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/121c689a7cad40e19c60664c8ff29e14, entries=200, sequenceid=14, filesize=38.6 K 2024-11-17T01:53:14,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/e4f4ed3caca2494dac8355798c52bf92 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/e4f4ed3caca2494dac8355798c52bf92 2024-11-17T01:53:14,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/e4f4ed3caca2494dac8355798c52bf92, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T01:53:14,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/ca84c2a725994c4a9517c200f98d9ed0 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/ca84c2a725994c4a9517c200f98d9ed0 2024-11-17T01:53:14,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/ca84c2a725994c4a9517c200f98d9ed0, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T01:53:14,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for fe092165677ecd92ff68c61a7627feb9 in 1399ms, sequenceid=14, compaction requested=false 2024-11-17T01:53:14,937 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T01:53:14,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:14,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-17T01:53:14,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:14,955 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-17T01:53:14,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:14,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:14,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:14,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:14,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:14,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:14,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b5e475f89ce74e71866c2a69d54f28a9_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_0/A:col10/1731808393659/Put/seqid=0 2024-11-17T01:53:14,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742153_1329 (size=12154) 2024-11-17T01:53:14,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,003 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b5e475f89ce74e71866c2a69d54f28a9_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b5e475f89ce74e71866c2a69d54f28a9_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:15,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/4f9491068c664ae9a0ea02e0684448f3, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:15,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/4f9491068c664ae9a0ea02e0684448f3 is 175, key is test_row_0/A:col10/1731808393659/Put/seqid=0 2024-11-17T01:53:15,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742154_1330 (size=30955) 2024-11-17T01:53:15,017 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/4f9491068c664ae9a0ea02e0684448f3 2024-11-17T01:53:15,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/314322d603594012be45fe8c0db274e1 is 50, key is test_row_0/B:col10/1731808393659/Put/seqid=0 2024-11-17T01:53:15,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742155_1331 (size=12001) 2024-11-17T01:53:15,452 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/314322d603594012be45fe8c0db274e1 2024-11-17T01:53:15,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/bc17e78b9d3849a1962cfe510ee71df3 is 50, key is test_row_0/C:col10/1731808393659/Put/seqid=0 2024-11-17T01:53:15,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742156_1332 (size=12001) 2024-11-17T01:53:15,480 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/bc17e78b9d3849a1962cfe510ee71df3 2024-11-17T01:53:15,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/4f9491068c664ae9a0ea02e0684448f3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/4f9491068c664ae9a0ea02e0684448f3 2024-11-17T01:53:15,491 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/4f9491068c664ae9a0ea02e0684448f3, entries=150, sequenceid=51, filesize=30.2 K 2024-11-17T01:53:15,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/314322d603594012be45fe8c0db274e1 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/314322d603594012be45fe8c0db274e1 2024-11-17T01:53:15,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,495 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/314322d603594012be45fe8c0db274e1, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T01:53:15,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/bc17e78b9d3849a1962cfe510ee71df3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/bc17e78b9d3849a1962cfe510ee71df3 2024-11-17T01:53:15,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,500 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/bc17e78b9d3849a1962cfe510ee71df3, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T01:53:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,501 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for fe092165677ecd92ff68c61a7627feb9 in 546ms, sequenceid=51, compaction requested=false 2024-11-17T01:53:15,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:15,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:15,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-17T01:53:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-17T01:53:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-17T01:53:15,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9320 sec 2024-11-17T01:53:15,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.9390 sec 2024-11-17T01:53:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T01:53:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,714 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,715 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:53:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-17T01:53:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T01:53:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,718 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:53:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,719 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:53:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:53:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T01:53:15,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-17T01:53:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:15,873 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-17T01:53:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:15,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:15,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:15,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:15,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:15,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d23163c2f08c4fd49affce46c5a71337_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_1/A:col10/1731808395862/Put/seqid=0 2024-11-17T01:53:15,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742157_1333 (size=9714) 2024-11-17T01:53:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,938 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d23163c2f08c4fd49affce46c5a71337_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d23163c2f08c4fd49affce46c5a71337_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/83e6a74b91e046be8fa4c70dce0237e8, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/83e6a74b91e046be8fa4c70dce0237e8 is 175, key is test_row_1/A:col10/1731808395862/Put/seqid=0 2024-11-17T01:53:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:15,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742158_1334 (size=22361) 2024-11-17T01:53:15,983 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/83e6a74b91e046be8fa4c70dce0237e8 2024-11-17T01:53:15,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/13599ccef142408dbd24493534554406 is 50, key is test_row_1/B:col10/1731808395862/Put/seqid=0 2024-11-17T01:53:16,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808395988, max timestamp=1731808395998Num uniques 859; 2024-11-17T01:53:16,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808395988, max timestamp=1731808395998Num uniques 859; 2024-11-17T01:53:16,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:53:16,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:53:16,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808395988, max timestamp=1731808395998Num uniques 859; 2024-11-17T01:53:16,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:53:16,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742159_1335 (size=9657) 2024-11-17T01:53:16,004 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/13599ccef142408dbd24493534554406 2024-11-17T01:53:16,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/a0cdde08ea4d41c5bb7b7c185b8a091a is 50, key is test_row_1/C:col10/1731808395862/Put/seqid=0 2024-11-17T01:53:16,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742160_1336 (size=9657) 2024-11-17T01:53:16,031 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/a0cdde08ea4d41c5bb7b7c185b8a091a 2024-11-17T01:53:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T01:53:16,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/83e6a74b91e046be8fa4c70dce0237e8 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/83e6a74b91e046be8fa4c70dce0237e8 2024-11-17T01:53:16,048 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/83e6a74b91e046be8fa4c70dce0237e8, entries=100, sequenceid=56, filesize=21.8 K 2024-11-17T01:53:16,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/13599ccef142408dbd24493534554406 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/13599ccef142408dbd24493534554406 2024-11-17T01:53:16,053 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/13599ccef142408dbd24493534554406, entries=100, sequenceid=56, filesize=9.4 K 2024-11-17T01:53:16,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/a0cdde08ea4d41c5bb7b7c185b8a091a as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/a0cdde08ea4d41c5bb7b7c185b8a091a 2024-11-17T01:53:16,060 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/a0cdde08ea4d41c5bb7b7c185b8a091a, entries=100, sequenceid=56, filesize=9.4 K 2024-11-17T01:53:16,061 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=140.89 KB/144270 for fe092165677ecd92ff68c61a7627feb9 in 188ms, sequenceid=56, compaction requested=true 2024-11-17T01:53:16,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:16,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:16,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-17T01:53:16,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-17T01:53:16,064 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-17T01:53:16,064 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 343 msec 2024-11-17T01:53:16,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 349 msec 2024-11-17T01:53:16,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731808396038, max timestamp=1731808396067Num uniques 1068; 2024-11-17T01:53:16,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731808396038, max timestamp=1731808396067Num uniques 1068; 2024-11-17T01:53:16,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-17T01:53:16,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-17T01:53:16,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731808396038, max timestamp=1731808396067Num uniques 1068; 2024-11-17T01:53:16,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-17T01:53:16,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:16,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=53.67 KB heapSize=140.39 KB 2024-11-17T01:53:16,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:16,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:16,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:16,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:16,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:16,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:16,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179765bb987b1147de9553fe0e9980ac83_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_0/A:col10/1731808396073/Put/seqid=0 2024-11-17T01:53:16,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742161_1337 (size=19474) 2024-11-17T01:53:16,114 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:16,124 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179765bb987b1147de9553fe0e9980ac83_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179765bb987b1147de9553fe0e9980ac83_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:16,125 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/1454d542dbd5477eb8bf1629d26a04b7, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:16,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/1454d542dbd5477eb8bf1629d26a04b7 is 175, key is test_row_0/A:col10/1731808396073/Put/seqid=0 2024-11-17T01:53:16,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731808396108, max timestamp=1731808396146Num uniques 859; 2024-11-17T01:53:16,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731808396108, max timestamp=1731808396146Num uniques 859; 2024-11-17T01:53:16,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T01:53:16,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T01:53:16,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731808396108, max timestamp=1731808396146Num uniques 859; 2024-11-17T01:53:16,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T01:53:16,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742162_1338 (size=56733) 2024-11-17T01:53:16,156 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/1454d542dbd5477eb8bf1629d26a04b7 2024-11-17T01:53:16,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/e7440811ba1e416c92bd6943a761245e is 50, key is test_row_0/B:col10/1731808396073/Put/seqid=0 2024-11-17T01:53:16,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742163_1339 (size=12001) 2024-11-17T01:53:16,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808456182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:16,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808456183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:16,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:16,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808456193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:16,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808456198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:16,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:16,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808456201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:16,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808456304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:16,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:16,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808456315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:16,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808456315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:16,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808456315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:16,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:16,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:16,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:16,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808456324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:16,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T01:53:16,345 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:16,345 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:53:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-17T01:53:16,347 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:53:16,356 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:53:16,356 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:53:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T01:53:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T01:53:16,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-17T01:53:16,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:16,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:16,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:16,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:16,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:16,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808456524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:16,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:16,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808456534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:16,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808456534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808456534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:16,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808456544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:16,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:16,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/e7440811ba1e416c92bd6943a761245e 2024-11-17T01:53:16,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/84c3bc6c7fb5429494ed317a31727e1c is 50, key is test_row_0/C:col10/1731808396073/Put/seqid=0 2024-11-17T01:53:16,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742164_1340 (size=12001) 2024-11-17T01:53:16,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/84c3bc6c7fb5429494ed317a31727e1c 2024-11-17T01:53:16,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/1454d542dbd5477eb8bf1629d26a04b7 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/1454d542dbd5477eb8bf1629d26a04b7 2024-11-17T01:53:16,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/1454d542dbd5477eb8bf1629d26a04b7, entries=300, sequenceid=97, filesize=55.4 K 2024-11-17T01:53:16,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/e7440811ba1e416c92bd6943a761245e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/e7440811ba1e416c92bd6943a761245e 2024-11-17T01:53:16,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/e7440811ba1e416c92bd6943a761245e, entries=150, sequenceid=97, filesize=11.7 K 2024-11-17T01:53:16,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/84c3bc6c7fb5429494ed317a31727e1c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/84c3bc6c7fb5429494ed317a31727e1c 2024-11-17T01:53:16,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/84c3bc6c7fb5429494ed317a31727e1c, entries=150, sequenceid=97, filesize=11.7 K 2024-11-17T01:53:16,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~140.34 KB/143712, currentSize=140.89 KB/144270 for fe092165677ecd92ff68c61a7627feb9 in 549ms, sequenceid=97, compaction requested=true 2024-11-17T01:53:16,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:16,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:53:16,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:16,625 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:53:16,625 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:53:16,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:53:16,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:16,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:53:16,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:16,627 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:53:16,627 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/B is initiating minor compaction (all files) 2024-11-17T01:53:16,627 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149598 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:53:16,627 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/A is initiating minor compaction (all files) 2024-11-17T01:53:16,627 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/A in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:16,627 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/121c689a7cad40e19c60664c8ff29e14, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/4f9491068c664ae9a0ea02e0684448f3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/83e6a74b91e046be8fa4c70dce0237e8, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/1454d542dbd5477eb8bf1629d26a04b7] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=146.1 K 2024-11-17T01:53:16,627 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:16,627 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/121c689a7cad40e19c60664c8ff29e14, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/4f9491068c664ae9a0ea02e0684448f3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/83e6a74b91e046be8fa4c70dce0237e8, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/1454d542dbd5477eb8bf1629d26a04b7] 2024-11-17T01:53:16,628 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/B in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:16,628 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/e4f4ed3caca2494dac8355798c52bf92, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/314322d603594012be45fe8c0db274e1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/13599ccef142408dbd24493534554406, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/e7440811ba1e416c92bd6943a761245e] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=44.6 K 2024-11-17T01:53:16,628 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 121c689a7cad40e19c60664c8ff29e14, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731808393523 2024-11-17T01:53:16,628 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting e4f4ed3caca2494dac8355798c52bf92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731808393523 2024-11-17T01:53:16,628 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f9491068c664ae9a0ea02e0684448f3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731808393656 2024-11-17T01:53:16,629 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83e6a74b91e046be8fa4c70dce0237e8, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731808395808 2024-11-17T01:53:16,629 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 314322d603594012be45fe8c0db274e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731808393656 2024-11-17T01:53:16,629 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 13599ccef142408dbd24493534554406, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731808395808 2024-11-17T01:53:16,629 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1454d542dbd5477eb8bf1629d26a04b7, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731808396038 2024-11-17T01:53:16,630 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting e7440811ba1e416c92bd6943a761245e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731808396038 2024-11-17T01:53:16,636 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:16,638 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#B#compaction#293 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:16,639 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/e226291e7dc142ae947a5f6b65fb2007 is 50, key is test_row_0/B:col10/1731808396073/Put/seqid=0 2024-11-17T01:53:16,650 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411176c35badfa669481591e576d23e986061_fe092165677ecd92ff68c61a7627feb9 store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:16,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742165_1341 (size=12139) 2024-11-17T01:53:16,654 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411176c35badfa669481591e576d23e986061_fe092165677ecd92ff68c61a7627feb9, store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:16,655 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176c35badfa669481591e576d23e986061_fe092165677ecd92ff68c61a7627feb9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:16,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-17T01:53:16,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:16,662 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-17T01:53:16,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:16,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:16,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:16,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:16,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:16,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:16,663 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/e226291e7dc142ae947a5f6b65fb2007 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/e226291e7dc142ae947a5f6b65fb2007 2024-11-17T01:53:16,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742166_1342 (size=4469) 2024-11-17T01:53:16,670 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#A#compaction#292 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:16,670 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/91130e9b5166416d851fea0271a2a4bf is 175, key is test_row_0/A:col10/1731808396073/Put/seqid=0 2024-11-17T01:53:16,672 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/B of fe092165677ecd92ff68c61a7627feb9 into e226291e7dc142ae947a5f6b65fb2007(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:16,672 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:16,672 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/B, priority=12, startTime=1731808396625; duration=0sec 2024-11-17T01:53:16,672 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:16,673 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:B 2024-11-17T01:53:16,673 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:53:16,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T01:53:16,675 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:53:16,675 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/C is initiating minor compaction (all files) 2024-11-17T01:53:16,675 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/C in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:16,675 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/ca84c2a725994c4a9517c200f98d9ed0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/bc17e78b9d3849a1962cfe510ee71df3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/a0cdde08ea4d41c5bb7b7c185b8a091a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/84c3bc6c7fb5429494ed317a31727e1c] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=44.6 K 2024-11-17T01:53:16,676 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting ca84c2a725994c4a9517c200f98d9ed0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731808393523 2024-11-17T01:53:16,676 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting bc17e78b9d3849a1962cfe510ee71df3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731808393656 2024-11-17T01:53:16,676 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting a0cdde08ea4d41c5bb7b7c185b8a091a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731808395808 2024-11-17T01:53:16,677 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 84c3bc6c7fb5429494ed317a31727e1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731808396038 2024-11-17T01:53:16,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b48603dc290a408b831ef089c62f4c62_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_0/A:col10/1731808396198/Put/seqid=0 2024-11-17T01:53:16,713 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#C#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:16,713 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/022ac1a7460a4ca2b7be140fefe2578c is 50, key is test_row_0/C:col10/1731808396073/Put/seqid=0 2024-11-17T01:53:16,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742167_1343 (size=31093) 2024-11-17T01:53:16,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742168_1344 (size=12304) 2024-11-17T01:53:16,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742169_1345 (size=12139) 2024-11-17T01:53:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:16,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:16,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808456874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:16,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:16,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808456874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:16,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:16,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808456876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:16,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:16,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808456883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:16,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:16,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:16,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808456903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:16,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:16,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T01:53:17,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808457016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:17,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808457016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:17,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808457016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:17,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808457017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:17,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:17,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808457024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:17,149 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/91130e9b5166416d851fea0271a2a4bf as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/91130e9b5166416d851fea0271a2a4bf 2024-11-17T01:53:17,155 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/A of fe092165677ecd92ff68c61a7627feb9 into 91130e9b5166416d851fea0271a2a4bf(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:17,155 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:17,155 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/A, priority=12, startTime=1731808396625; duration=0sec 2024-11-17T01:53:17,156 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:17,156 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:A 2024-11-17T01:53:17,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:17,178 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b48603dc290a408b831ef089c62f4c62_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b48603dc290a408b831ef089c62f4c62_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:17,179 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/022ac1a7460a4ca2b7be140fefe2578c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/022ac1a7460a4ca2b7be140fefe2578c 2024-11-17T01:53:17,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/35fc2285351546a7bdbdf54e18656db0, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:17,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/35fc2285351546a7bdbdf54e18656db0 is 175, key is test_row_0/A:col10/1731808396198/Put/seqid=0 2024-11-17T01:53:17,187 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/C of fe092165677ecd92ff68c61a7627feb9 into 022ac1a7460a4ca2b7be140fefe2578c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:17,187 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:17,187 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/C, priority=12, startTime=1731808396626; duration=0sec 2024-11-17T01:53:17,187 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:17,187 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:C 2024-11-17T01:53:17,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742170_1346 (size=31105) 2024-11-17T01:53:17,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:17,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808457234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:17,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:17,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:17,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808457234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:17,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:17,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808457234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:17,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:17,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808457244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:17,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:17,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:17,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:17,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808457255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T01:53:17,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:17,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808457554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:17,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:17,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808457554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:17,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:17,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:17,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808457554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:17,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:17,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:17,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808457554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:17,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:17,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:17,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:17,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808457574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:17,597 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/35fc2285351546a7bdbdf54e18656db0 2024-11-17T01:53:17,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/5ad7950f20604d69aff208cb88040a2d is 50, key is test_row_0/B:col10/1731808396198/Put/seqid=0 2024-11-17T01:53:17,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742171_1347 (size=12151) 2024-11-17T01:53:18,015 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/5ad7950f20604d69aff208cb88040a2d 2024-11-17T01:53:18,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/b0a0f80db2bb49cf96cdf924411b537b is 50, key is test_row_0/C:col10/1731808396198/Put/seqid=0 2024-11-17T01:53:18,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742172_1348 (size=12151) 2024-11-17T01:53:18,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808458074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:18,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:18,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:18,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:18,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808458074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:18,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:18,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:18,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:18,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808458074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:18,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:18,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:18,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:18,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808458074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:18,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808458094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:18,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:18,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:18,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:18,426 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/b0a0f80db2bb49cf96cdf924411b537b 2024-11-17T01:53:18,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/35fc2285351546a7bdbdf54e18656db0 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/35fc2285351546a7bdbdf54e18656db0 2024-11-17T01:53:18,434 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/35fc2285351546a7bdbdf54e18656db0, entries=150, sequenceid=132, filesize=30.4 K 2024-11-17T01:53:18,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/5ad7950f20604d69aff208cb88040a2d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/5ad7950f20604d69aff208cb88040a2d 2024-11-17T01:53:18,440 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/5ad7950f20604d69aff208cb88040a2d, entries=150, sequenceid=132, filesize=11.9 K 2024-11-17T01:53:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/b0a0f80db2bb49cf96cdf924411b537b as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/b0a0f80db2bb49cf96cdf924411b537b 2024-11-17T01:53:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,445 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/b0a0f80db2bb49cf96cdf924411b537b, entries=150, sequenceid=132, filesize=11.9 K 2024-11-17T01:53:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,446 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for fe092165677ecd92ff68c61a7627feb9 in 1784ms, sequenceid=132, compaction requested=false 2024-11-17T01:53:18,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:18,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-17T01:53:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-17T01:53:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-17T01:53:18,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0920 sec 2024-11-17T01:53:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,452 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.1060 sec 2024-11-17T01:53:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T01:53:18,505 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:18,505 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:53:18,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-17T01:53:18,507 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:53:18,508 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:53:18,508 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:53:18,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T01:53:18,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T01:53:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T01:53:18,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:18,660 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T01:53:18,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:18,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:18,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:18,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:18,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:18,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a0735bed237c4314ae6536a948dc8590_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_0/A:col10/1731808396901/Put/seqid=0 2024-11-17T01:53:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742173_1349 (size=9814) 2024-11-17T01:53:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T01:53:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,099 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a0735bed237c4314ae6536a948dc8590_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a0735bed237c4314ae6536a948dc8590_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/2c701c8ad7f84cc48be996671ed98d20, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:19,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/2c701c8ad7f84cc48be996671ed98d20 is 175, key is test_row_0/A:col10/1731808396901/Put/seqid=0 2024-11-17T01:53:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T01:53:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742174_1350 (size=22461) 2024-11-17T01:53:19,148 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/2c701c8ad7f84cc48be996671ed98d20 2024-11-17T01:53:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/b1c3f5fb848a426fbbbfc6cd7b6f0a6d is 50, key is test_row_0/B:col10/1731808396901/Put/seqid=0 2024-11-17T01:53:19,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742175_1351 (size=9757) 2024-11-17T01:53:19,229 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/b1c3f5fb848a426fbbbfc6cd7b6f0a6d 2024-11-17T01:53:19,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:19,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731808399200, max timestamp=1731808399236Num uniques 859; 2024-11-17T01:53:19,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T01:53:19,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731808399200, max timestamp=1731808399236Num uniques 859; 2024-11-17T01:53:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808459209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:19,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T01:53:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:19,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731808399200, max timestamp=1731808399236Num uniques 859; 2024-11-17T01:53:19,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T01:53:19,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/dc199d542ec54d71936dd548a6f9e2b0 is 50, key is test_row_0/C:col10/1731808396901/Put/seqid=0 2024-11-17T01:53:19,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742176_1352 (size=9757) 2024-11-17T01:53:19,291 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/dc199d542ec54d71936dd548a6f9e2b0 2024-11-17T01:53:19,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/2c701c8ad7f84cc48be996671ed98d20 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/2c701c8ad7f84cc48be996671ed98d20 2024-11-17T01:53:19,302 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/2c701c8ad7f84cc48be996671ed98d20, entries=100, sequenceid=145, filesize=21.9 K 2024-11-17T01:53:19,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/b1c3f5fb848a426fbbbfc6cd7b6f0a6d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/b1c3f5fb848a426fbbbfc6cd7b6f0a6d 2024-11-17T01:53:19,307 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/b1c3f5fb848a426fbbbfc6cd7b6f0a6d, entries=100, sequenceid=145, filesize=9.5 K 2024-11-17T01:53:19,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/dc199d542ec54d71936dd548a6f9e2b0 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/dc199d542ec54d71936dd548a6f9e2b0 2024-11-17T01:53:19,313 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/dc199d542ec54d71936dd548a6f9e2b0, entries=100, sequenceid=145, filesize=9.5 K 2024-11-17T01:53:19,314 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=100.63 KB/103050 for fe092165677ecd92ff68c61a7627feb9 in 654ms, sequenceid=145, compaction requested=true 2024-11-17T01:53:19,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:19,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:19,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-17T01:53:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:19,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=107.34 KB heapSize=295.92 KB 2024-11-17T01:53:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-17T01:53:19,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:19,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:19,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:19,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:19,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:19,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:19,318 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-17T01:53:19,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 808 msec 2024-11-17T01:53:19,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 814 msec 2024-11-17T01:53:19,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111700b397db2d2345eaac07e748c4d22348_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_0/A:col10/1731808399293/Put/seqid=0 2024-11-17T01:53:19,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808459353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:19,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808459355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:19,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808459370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:19,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808459372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:19,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742177_1353 (size=14794) 2024-11-17T01:53:19,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808459382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:19,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:19,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808459485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:19,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:19,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808459485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:19,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:19,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:19,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:19,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808459504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:19,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808459504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:19,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:19,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808459517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:19,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:19,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:19,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T01:53:19,645 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:19,645 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:53:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-17T01:53:19,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T01:53:19,648 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:53:19,648 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:53:19,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:53:19,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808459709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:19,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808459709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:19,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:19,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808459724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:19,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808459735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:19,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808459735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T01:53:19,802 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:19,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T01:53:19,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:19,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:19,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:19,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:19,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:19,813 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111700b397db2d2345eaac07e748c4d22348_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111700b397db2d2345eaac07e748c4d22348_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:19,814 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/35c892eaab4541dabc4169cec1064873, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:19,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/35c892eaab4541dabc4169cec1064873 is 175, key is test_row_0/A:col10/1731808399293/Put/seqid=0 2024-11-17T01:53:19,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742178_1354 (size=39749) 2024-11-17T01:53:19,830 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/35c892eaab4541dabc4169cec1064873 2024-11-17T01:53:19,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/95e8fabb18b342aabd610722a8467b56 is 50, key is test_row_0/B:col10/1731808399293/Put/seqid=0 2024-11-17T01:53:19,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742179_1355 (size=12151) 2024-11-17T01:53:19,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/95e8fabb18b342aabd610722a8467b56 2024-11-17T01:53:19,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/50bea256784948848a21686766f5df0a is 50, key is test_row_0/C:col10/1731808399293/Put/seqid=0 2024-11-17T01:53:19,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742180_1356 (size=12151) 2024-11-17T01:53:19,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/50bea256784948848a21686766f5df0a 2024-11-17T01:53:19,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/35c892eaab4541dabc4169cec1064873 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/35c892eaab4541dabc4169cec1064873 2024-11-17T01:53:19,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/35c892eaab4541dabc4169cec1064873, entries=200, sequenceid=176, filesize=38.8 K 2024-11-17T01:53:19,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/95e8fabb18b342aabd610722a8467b56 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/95e8fabb18b342aabd610722a8467b56 2024-11-17T01:53:19,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/95e8fabb18b342aabd610722a8467b56, entries=150, sequenceid=176, filesize=11.9 K 2024-11-17T01:53:19,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/50bea256784948848a21686766f5df0a as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/50bea256784948848a21686766f5df0a 2024-11-17T01:53:19,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/50bea256784948848a21686766f5df0a, entries=150, sequenceid=176, filesize=11.9 K 2024-11-17T01:53:19,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~295.88 KB/302976, currentSize=73.80 KB/75570 for fe092165677ecd92ff68c61a7627feb9 in 605ms, sequenceid=176, compaction requested=true 2024-11-17T01:53:19,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:19,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:53:19,921 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:53:19,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:19,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:53:19,922 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:53:19,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:19,922 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124408 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:53:19,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:53:19,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:19,922 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/A is initiating minor compaction (all files) 2024-11-17T01:53:19,922 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/A in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:19,922 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/91130e9b5166416d851fea0271a2a4bf, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/35fc2285351546a7bdbdf54e18656db0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/2c701c8ad7f84cc48be996671ed98d20, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/35c892eaab4541dabc4169cec1064873] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=121.5 K 2024-11-17T01:53:19,922 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:19,922 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/91130e9b5166416d851fea0271a2a4bf, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/35fc2285351546a7bdbdf54e18656db0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/2c701c8ad7f84cc48be996671ed98d20, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/35c892eaab4541dabc4169cec1064873] 2024-11-17T01:53:19,923 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91130e9b5166416d851fea0271a2a4bf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731808396038 2024-11-17T01:53:19,923 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35fc2285351546a7bdbdf54e18656db0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731808396190 2024-11-17T01:53:19,924 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c701c8ad7f84cc48be996671ed98d20, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731808396871 2024-11-17T01:53:19,924 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:53:19,924 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/B is initiating minor compaction (all files) 2024-11-17T01:53:19,924 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/B in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:19,924 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/e226291e7dc142ae947a5f6b65fb2007, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/5ad7950f20604d69aff208cb88040a2d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/b1c3f5fb848a426fbbbfc6cd7b6f0a6d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/95e8fabb18b342aabd610722a8467b56] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=45.1 K 2024-11-17T01:53:19,924 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35c892eaab4541dabc4169cec1064873, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731808399285 2024-11-17T01:53:19,927 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting e226291e7dc142ae947a5f6b65fb2007, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731808396038 2024-11-17T01:53:19,928 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ad7950f20604d69aff208cb88040a2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731808396190 2024-11-17T01:53:19,928 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting b1c3f5fb848a426fbbbfc6cd7b6f0a6d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731808396871 2024-11-17T01:53:19,929 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 95e8fabb18b342aabd610722a8467b56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731808399290 2024-11-17T01:53:19,943 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:19,953 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#B#compaction#305 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:19,953 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/0f051c7c89b74bdeb1bcc6d0327afa9c is 50, key is test_row_0/B:col10/1731808399293/Put/seqid=0 2024-11-17T01:53:19,954 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411170e416c6d5a6b4ae8824252f5f4a90200_fe092165677ecd92ff68c61a7627feb9 store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:19,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T01:53:19,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:19,956 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T01:53:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T01:53:19,965 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411170e416c6d5a6b4ae8824252f5f4a90200_fe092165677ecd92ff68c61a7627feb9, store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:19,965 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170e416c6d5a6b4ae8824252f5f4a90200_fe092165677ecd92ff68c61a7627feb9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:19,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742181_1357 (size=12425) 2024-11-17T01:53:19,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172491ecbcb90346e9adad7a8ca111f6ff_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_0/A:col10/1731808399378/Put/seqid=0 2024-11-17T01:53:19,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742182_1358 (size=4469) 2024-11-17T01:53:19,999 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#A#compaction#304 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:20,000 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/2fe06fd186b64052bb8b36253c625bf0 is 175, key is test_row_0/A:col10/1731808399293/Put/seqid=0 2024-11-17T01:53:20,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742183_1359 (size=12304) 2024-11-17T01:53:20,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742184_1360 (size=31379) 2024-11-17T01:53:20,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,013 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172491ecbcb90346e9adad7a8ca111f6ff_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172491ecbcb90346e9adad7a8ca111f6ff_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:20,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/1b3a85bbb20041c492719402b5993115, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:20,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/1b3a85bbb20041c492719402b5993115 is 175, key is test_row_0/A:col10/1731808399378/Put/seqid=0 2024-11-17T01:53:20,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742185_1361 (size=31105) 2024-11-17T01:53:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:20,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:20,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808460074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:20,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808460085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:20,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808460088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:20,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:20,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:20,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808460089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:20,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:20,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808460090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:20,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:20,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:20,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808460194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:20,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:20,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808460204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:20,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808460204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:20,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808460214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:20,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:20,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808460214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:20,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T01:53:20,392 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/0f051c7c89b74bdeb1bcc6d0327afa9c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/0f051c7c89b74bdeb1bcc6d0327afa9c 2024-11-17T01:53:20,396 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/B of fe092165677ecd92ff68c61a7627feb9 into 0f051c7c89b74bdeb1bcc6d0327afa9c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:20,396 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:20,396 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/B, priority=12, startTime=1731808399922; duration=0sec 2024-11-17T01:53:20,396 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:20,396 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:B 2024-11-17T01:53:20,396 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:53:20,397 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:53:20,397 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/C is initiating minor compaction (all files) 2024-11-17T01:53:20,397 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/C in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:20,397 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/022ac1a7460a4ca2b7be140fefe2578c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/b0a0f80db2bb49cf96cdf924411b537b, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/dc199d542ec54d71936dd548a6f9e2b0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/50bea256784948848a21686766f5df0a] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=45.1 K 2024-11-17T01:53:20,398 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 022ac1a7460a4ca2b7be140fefe2578c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731808396038 2024-11-17T01:53:20,398 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting b0a0f80db2bb49cf96cdf924411b537b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731808396190 2024-11-17T01:53:20,398 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting dc199d542ec54d71936dd548a6f9e2b0, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731808396871 2024-11-17T01:53:20,402 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 50bea256784948848a21686766f5df0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731808399290 2024-11-17T01:53:20,409 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/2fe06fd186b64052bb8b36253c625bf0 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/2fe06fd186b64052bb8b36253c625bf0 2024-11-17T01:53:20,415 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#C#compaction#307 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:20,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808460404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:20,415 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/976141a2490d46bdb0b16630448b2de1 is 50, key is test_row_0/C:col10/1731808399293/Put/seqid=0 2024-11-17T01:53:20,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:20,417 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/A of fe092165677ecd92ff68c61a7627feb9 into 2fe06fd186b64052bb8b36253c625bf0(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:20,417 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:20,417 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/A, priority=12, startTime=1731808399920; duration=0sec 2024-11-17T01:53:20,417 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:20,417 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:A 2024-11-17T01:53:20,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742186_1362 (size=12425) 2024-11-17T01:53:20,429 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/1b3a85bbb20041c492719402b5993115 2024-11-17T01:53:20,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808460424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:20,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808460424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:20,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/c7f7350843b44bb2b96a408100918e29 is 50, key is test_row_0/B:col10/1731808399378/Put/seqid=0 2024-11-17T01:53:20,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808460434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:20,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808460434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:20,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:20,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742187_1363 (size=12151) 2024-11-17T01:53:20,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808460724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:20,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808460744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808460744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:20,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:20,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:20,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808460744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:20,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808460756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:20,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:20,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:20,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T01:53:20,834 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/976141a2490d46bdb0b16630448b2de1 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/976141a2490d46bdb0b16630448b2de1 2024-11-17T01:53:20,841 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/C of fe092165677ecd92ff68c61a7627feb9 into 976141a2490d46bdb0b16630448b2de1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:20,841 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:20,841 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/C, priority=12, startTime=1731808399922; duration=0sec 2024-11-17T01:53:20,841 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:20,841 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:C 2024-11-17T01:53:20,849 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/c7f7350843b44bb2b96a408100918e29 2024-11-17T01:53:20,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/cdcc03265ed84b0c8a06e8f7f87c61df is 50, key is test_row_0/C:col10/1731808399378/Put/seqid=0 2024-11-17T01:53:20,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742188_1364 (size=12151) 2024-11-17T01:53:20,874 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/cdcc03265ed84b0c8a06e8f7f87c61df 2024-11-17T01:53:20,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/1b3a85bbb20041c492719402b5993115 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/1b3a85bbb20041c492719402b5993115 2024-11-17T01:53:20,890 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/1b3a85bbb20041c492719402b5993115, entries=150, sequenceid=190, filesize=30.4 K 2024-11-17T01:53:20,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/c7f7350843b44bb2b96a408100918e29 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/c7f7350843b44bb2b96a408100918e29 2024-11-17T01:53:20,896 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/c7f7350843b44bb2b96a408100918e29, entries=150, sequenceid=190, filesize=11.9 K 2024-11-17T01:53:20,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/cdcc03265ed84b0c8a06e8f7f87c61df as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/cdcc03265ed84b0c8a06e8f7f87c61df 2024-11-17T01:53:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,905 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/cdcc03265ed84b0c8a06e8f7f87c61df, entries=150, sequenceid=190, filesize=11.9 K 2024-11-17T01:53:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,906 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for fe092165677ecd92ff68c61a7627feb9 in 950ms, sequenceid=190, compaction requested=false 2024-11-17T01:53:20,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:20,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:20,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-17T01:53:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-17T01:53:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-17T01:53:20,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2600 sec 2024-11-17T01:53:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.2670 sec 2024-11-17T01:53:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:21,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T01:53:21,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:21,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:21,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:21,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:21,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:21,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:21,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175838305bebb34024b7c8c1cd84adbf67_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_0/A:col10/1731808401266/Put/seqid=0 2024-11-17T01:53:21,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742189_1365 (size=12304) 2024-11-17T01:53:21,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:21,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808461308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:21,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:21,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808461308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:21,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:21,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:21,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808461309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:21,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:21,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808461320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:21,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:21,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808461322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:21,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808461424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:21,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808461434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:21,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:21,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808461435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:21,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:21,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808461435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:21,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:21,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808461444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:21,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:21,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808461634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:21,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:21,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808461646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:21,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:21,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:21,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808461654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:21,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:21,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808461654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:21,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808461655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:21,701 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:21,706 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175838305bebb34024b7c8c1cd84adbf67_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175838305bebb34024b7c8c1cd84adbf67_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:21,707 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/71f7fe8b593e405cac1a303ca004dbeb, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:21,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/71f7fe8b593e405cac1a303ca004dbeb is 175, key is test_row_0/A:col10/1731808401266/Put/seqid=0 2024-11-17T01:53:21,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742190_1366 (size=31101) 2024-11-17T01:53:21,746 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/71f7fe8b593e405cac1a303ca004dbeb 2024-11-17T01:53:21,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/ff81058e6be4437fa87411f498980f9d is 50, key is test_row_0/B:col10/1731808401266/Put/seqid=0 2024-11-17T01:53:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T01:53:21,804 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:21,805 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:53:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-17T01:53:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T01:53:21,807 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:53:21,808 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:53:21,808 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:53:21,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742191_1367 (size=9757) 2024-11-17T01:53:21,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/ff81058e6be4437fa87411f498980f9d 2024-11-17T01:53:21,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/2a85c94abf5a498bb4becdf18e1b9255 is 50, key is test_row_0/C:col10/1731808401266/Put/seqid=0 2024-11-17T01:53:21,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742192_1368 (size=9757) 2024-11-17T01:53:21,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/2a85c94abf5a498bb4becdf18e1b9255 2024-11-17T01:53:21,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/71f7fe8b593e405cac1a303ca004dbeb as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/71f7fe8b593e405cac1a303ca004dbeb 2024-11-17T01:53:21,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/71f7fe8b593e405cac1a303ca004dbeb, entries=150, sequenceid=212, filesize=30.4 K 2024-11-17T01:53:21,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/ff81058e6be4437fa87411f498980f9d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/ff81058e6be4437fa87411f498980f9d 2024-11-17T01:53:21,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/ff81058e6be4437fa87411f498980f9d, entries=100, sequenceid=212, filesize=9.5 K 2024-11-17T01:53:21,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/2a85c94abf5a498bb4becdf18e1b9255 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/2a85c94abf5a498bb4becdf18e1b9255 2024-11-17T01:53:21,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/2a85c94abf5a498bb4becdf18e1b9255, entries=100, sequenceid=212, filesize=9.5 K 2024-11-17T01:53:21,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for fe092165677ecd92ff68c61a7627feb9 in 642ms, sequenceid=212, compaction requested=true 2024-11-17T01:53:21,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:21,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:53:21,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:21,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:53:21,912 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:21,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:21,912 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:21,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:53:21,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:21,912 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:21,913 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/A is initiating minor compaction (all files) 2024-11-17T01:53:21,913 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/A in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:21,913 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/2fe06fd186b64052bb8b36253c625bf0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/1b3a85bbb20041c492719402b5993115, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/71f7fe8b593e405cac1a303ca004dbeb] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=91.4 K 2024-11-17T01:53:21,913 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:21,913 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/2fe06fd186b64052bb8b36253c625bf0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/1b3a85bbb20041c492719402b5993115, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/71f7fe8b593e405cac1a303ca004dbeb] 2024-11-17T01:53:21,913 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:21,913 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/B is initiating minor compaction (all files) 2024-11-17T01:53:21,913 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/B in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:21,913 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/0f051c7c89b74bdeb1bcc6d0327afa9c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/c7f7350843b44bb2b96a408100918e29, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/ff81058e6be4437fa87411f498980f9d] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=33.5 K 2024-11-17T01:53:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T01:53:21,914 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fe06fd186b64052bb8b36253c625bf0, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731808399290 2024-11-17T01:53:21,914 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f051c7c89b74bdeb1bcc6d0327afa9c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731808399290 2024-11-17T01:53:21,914 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b3a85bbb20041c492719402b5993115, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731808399344 2024-11-17T01:53:21,914 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting c7f7350843b44bb2b96a408100918e29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731808399344 2024-11-17T01:53:21,917 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting ff81058e6be4437fa87411f498980f9d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731808400089 2024-11-17T01:53:21,917 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71f7fe8b593e405cac1a303ca004dbeb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731808400085 2024-11-17T01:53:21,940 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#B#compaction#313 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:21,941 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/dfaee356cb38486aaf3c4929c1037469 is 50, key is test_row_0/B:col10/1731808401266/Put/seqid=0 2024-11-17T01:53:21,946 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:21,960 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117ced9117fcb134c06a28a0455780e9360_fe092165677ecd92ff68c61a7627feb9 store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:21,962 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117ced9117fcb134c06a28a0455780e9360_fe092165677ecd92ff68c61a7627feb9, store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:21,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T01:53:21,962 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ced9117fcb134c06a28a0455780e9360_fe092165677ecd92ff68c61a7627feb9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:21,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T01:53:21,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:21,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:21,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:21,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:21,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:21,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:21,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:21,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:21,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:21,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ef27531a4a7d4ee1a6fb1996a6ed3bc5_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_0/A:col10/1731808401318/Put/seqid=0 2024-11-17T01:53:21,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742193_1369 (size=12527) 2024-11-17T01:53:22,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742194_1370 (size=4469) 2024-11-17T01:53:22,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742195_1371 (size=14794) 2024-11-17T01:53:22,016 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:22,021 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ef27531a4a7d4ee1a6fb1996a6ed3bc5_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ef27531a4a7d4ee1a6fb1996a6ed3bc5_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:22,022 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/ee23128a51b445339866926d39cd5413, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:22,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/ee23128a51b445339866926d39cd5413 is 175, key is test_row_0/A:col10/1731808401318/Put/seqid=0 2024-11-17T01:53:22,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808462012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742196_1372 (size=39749) 2024-11-17T01:53:22,045 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/ee23128a51b445339866926d39cd5413 2024-11-17T01:53:22,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808462032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808462033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808462034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808462034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/a6513d5187ca428cbb542e10604d8389 is 50, key is test_row_0/B:col10/1731808401318/Put/seqid=0 2024-11-17T01:53:22,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742197_1373 (size=12151) 2024-11-17T01:53:22,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/a6513d5187ca428cbb542e10604d8389 2024-11-17T01:53:22,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/130d9ed4e36e40b8a59580a41f0db6f5 is 50, key is test_row_0/C:col10/1731808401318/Put/seqid=0 2024-11-17T01:53:22,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T01:53:22,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:22,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:22,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:22,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:22,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T01:53:22,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808462144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808462155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808462155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808462155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742198_1374 (size=12151) 2024-11-17T01:53:22,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/130d9ed4e36e40b8a59580a41f0db6f5 2024-11-17T01:53:22,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808462166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/ee23128a51b445339866926d39cd5413 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/ee23128a51b445339866926d39cd5413 2024-11-17T01:53:22,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/ee23128a51b445339866926d39cd5413, entries=200, sequenceid=226, filesize=38.8 K 2024-11-17T01:53:22,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/a6513d5187ca428cbb542e10604d8389 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/a6513d5187ca428cbb542e10604d8389 2024-11-17T01:53:22,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/a6513d5187ca428cbb542e10604d8389, entries=150, sequenceid=226, filesize=11.9 K 2024-11-17T01:53:22,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/130d9ed4e36e40b8a59580a41f0db6f5 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/130d9ed4e36e40b8a59580a41f0db6f5 2024-11-17T01:53:22,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/130d9ed4e36e40b8a59580a41f0db6f5, entries=150, sequenceid=226, filesize=11.9 K 2024-11-17T01:53:22,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for fe092165677ecd92ff68c61a7627feb9 in 248ms, sequenceid=226, compaction requested=true 2024-11-17T01:53:22,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:22,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:A, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:53:22,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T01:53:22,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:B, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:53:22,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-17T01:53:22,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:53:22,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-17T01:53:22,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T01:53:22,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:22,274 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T01:53:22,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:22,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:22,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:22,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:22,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:22,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:22,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111754f27d6c6b4f41aa9bb9f88dea5e59c8_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_0/A:col10/1731808402033/Put/seqid=0 2024-11-17T01:53:22,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742199_1375 (size=12304) 2024-11-17T01:53:22,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:22,361 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111754f27d6c6b4f41aa9bb9f88dea5e59c8_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111754f27d6c6b4f41aa9bb9f88dea5e59c8_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:22,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/c9b5ee40a07847279edaaa7fa6bb9838, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:22,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/c9b5ee40a07847279edaaa7fa6bb9838 is 175, key is test_row_0/A:col10/1731808402033/Put/seqid=0 2024-11-17T01:53:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:22,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:22,403 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/dfaee356cb38486aaf3c4929c1037469 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/dfaee356cb38486aaf3c4929c1037469 2024-11-17T01:53:22,403 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#A#compaction#314 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:22,404 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/33cf21cd8dfc4067990ba3aca6ec892c is 175, key is test_row_0/A:col10/1731808401266/Put/seqid=0 2024-11-17T01:53:22,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742200_1376 (size=31105) 2024-11-17T01:53:22,422 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/B of fe092165677ecd92ff68c61a7627feb9 into dfaee356cb38486aaf3c4929c1037469(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:22,422 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:22,423 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/B, priority=13, startTime=1731808401912; duration=0sec 2024-11-17T01:53:22,423 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-17T01:53:22,423 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:B 2024-11-17T01:53:22,423 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-17T01:53:22,423 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-17T01:53:22,423 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-17T01:53:22,424 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. because compaction request was cancelled 2024-11-17T01:53:22,424 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:A 2024-11-17T01:53:22,424 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:53:22,425 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46484 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:53:22,425 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/C is initiating minor compaction (all files) 2024-11-17T01:53:22,425 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/C in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:22,425 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/976141a2490d46bdb0b16630448b2de1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/cdcc03265ed84b0c8a06e8f7f87c61df, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/2a85c94abf5a498bb4becdf18e1b9255, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/130d9ed4e36e40b8a59580a41f0db6f5] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=45.4 K 2024-11-17T01:53:22,425 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 976141a2490d46bdb0b16630448b2de1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731808399290 2024-11-17T01:53:22,426 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting cdcc03265ed84b0c8a06e8f7f87c61df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731808399344 2024-11-17T01:53:22,426 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a85c94abf5a498bb4becdf18e1b9255, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731808400089 2024-11-17T01:53:22,426 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 130d9ed4e36e40b8a59580a41f0db6f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731808401317 2024-11-17T01:53:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T01:53:22,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808462410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808462415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808462416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808462417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742201_1377 (size=31588) 2024-11-17T01:53:22,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808462438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,462 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/33cf21cd8dfc4067990ba3aca6ec892c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/33cf21cd8dfc4067990ba3aca6ec892c 2024-11-17T01:53:22,469 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/A of fe092165677ecd92ff68c61a7627feb9 into 33cf21cd8dfc4067990ba3aca6ec892c(size=30.8 K), total size for store is 69.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:22,470 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:22,470 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/A, priority=13, startTime=1731808401911; duration=0sec 2024-11-17T01:53:22,470 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T01:53:22,470 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:A 2024-11-17T01:53:22,470 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:B 2024-11-17T01:53:22,470 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-17T01:53:22,470 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-17T01:53:22,470 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-17T01:53:22,470 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. because compaction request was cancelled 2024-11-17T01:53:22,470 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:C 2024-11-17T01:53:22,470 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-17T01:53:22,471 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-17T01:53:22,471 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-17T01:53:22,471 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. because compaction request was cancelled 2024-11-17T01:53:22,471 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:B 2024-11-17T01:53:22,472 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#C#compaction#319 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:22,473 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/db24bb9ac28648f482b25f5f226c0e8a is 50, key is test_row_0/C:col10/1731808401318/Put/seqid=0 2024-11-17T01:53:22,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742202_1378 (size=12561) 2024-11-17T01:53:22,495 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/db24bb9ac28648f482b25f5f226c0e8a as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/db24bb9ac28648f482b25f5f226c0e8a 2024-11-17T01:53:22,501 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/C of fe092165677ecd92ff68c61a7627feb9 into db24bb9ac28648f482b25f5f226c0e8a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:22,501 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:22,501 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/C, priority=12, startTime=1731808402211; duration=0sec 2024-11-17T01:53:22,501 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:22,501 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:C 2024-11-17T01:53:22,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808462544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808462544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808462544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808462545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808462564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808462764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808462764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808462764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808462764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808462774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:22,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:22,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:22,820 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/c9b5ee40a07847279edaaa7fa6bb9838 2024-11-17T01:53:22,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/60d4197e5a6a43789832f9d163b3b6b1 is 50, key is test_row_0/B:col10/1731808402033/Put/seqid=0 2024-11-17T01:53:22,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742203_1379 (size=12151) 2024-11-17T01:53:22,859 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/60d4197e5a6a43789832f9d163b3b6b1 2024-11-17T01:53:22,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/583adb4bf2284d64b296b9a28215d9d9 is 50, key is test_row_0/C:col10/1731808402033/Put/seqid=0 2024-11-17T01:53:22,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742204_1380 (size=12151) 2024-11-17T01:53:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T01:53:23,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:23,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808463084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808463084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:23,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:23,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808463085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808463085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:23,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808463094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:23,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:23,291 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/583adb4bf2284d64b296b9a28215d9d9 2024-11-17T01:53:23,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/c9b5ee40a07847279edaaa7fa6bb9838 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/c9b5ee40a07847279edaaa7fa6bb9838 2024-11-17T01:53:23,301 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/c9b5ee40a07847279edaaa7fa6bb9838, entries=150, sequenceid=245, filesize=30.4 K 2024-11-17T01:53:23,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/60d4197e5a6a43789832f9d163b3b6b1 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/60d4197e5a6a43789832f9d163b3b6b1 2024-11-17T01:53:23,305 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/60d4197e5a6a43789832f9d163b3b6b1, entries=150, sequenceid=245, filesize=11.9 K 2024-11-17T01:53:23,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/583adb4bf2284d64b296b9a28215d9d9 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/583adb4bf2284d64b296b9a28215d9d9 2024-11-17T01:53:23,317 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/583adb4bf2284d64b296b9a28215d9d9, entries=150, sequenceid=245, filesize=11.9 K 2024-11-17T01:53:23,318 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for fe092165677ecd92ff68c61a7627feb9 in 1044ms, sequenceid=245, compaction requested=true 2024-11-17T01:53:23,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:23,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:23,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-17T01:53:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-17T01:53:23,322 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-17T01:53:23,322 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5120 sec 2024-11-17T01:53:23,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.5180 sec 2024-11-17T01:53:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:23,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T01:53:23,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:23,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:23,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:23,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:23,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:23,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:23,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b29587cac3794444bc32d79cd727c7be_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_0/A:col10/1731808402385/Put/seqid=0 2024-11-17T01:53:23,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742205_1381 (size=17434) 2024-11-17T01:53:23,652 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:23,657 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b29587cac3794444bc32d79cd727c7be_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b29587cac3794444bc32d79cd727c7be_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:23,658 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/88bf822f1e1f480bbee5017bc4c2f35d, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:23,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808463647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:23,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/88bf822f1e1f480bbee5017bc4c2f35d is 175, key is test_row_0/A:col10/1731808402385/Put/seqid=0 2024-11-17T01:53:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:23,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808463651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:23,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808463654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:23,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:23,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808463656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:23,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:23,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:23,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808463658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:23,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:23,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742206_1382 (size=48539) 2024-11-17T01:53:23,684 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/88bf822f1e1f480bbee5017bc4c2f35d 2024-11-17T01:53:23,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/0a0a829c9e3b4637adb426c439c36a27 is 50, key is test_row_0/B:col10/1731808402385/Put/seqid=0 2024-11-17T01:53:23,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742207_1383 (size=12251) 2024-11-17T01:53:23,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/0a0a829c9e3b4637adb426c439c36a27 2024-11-17T01:53:23,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/564a4b288fa449ffa7e90f8834bd925f is 50, key is test_row_0/C:col10/1731808402385/Put/seqid=0 2024-11-17T01:53:23,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742208_1384 (size=12251) 2024-11-17T01:53:23,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/564a4b288fa449ffa7e90f8834bd925f 2024-11-17T01:53:23,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/88bf822f1e1f480bbee5017bc4c2f35d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/88bf822f1e1f480bbee5017bc4c2f35d 2024-11-17T01:53:23,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/88bf822f1e1f480bbee5017bc4c2f35d, entries=250, sequenceid=263, filesize=47.4 K 2024-11-17T01:53:23,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/0a0a829c9e3b4637adb426c439c36a27 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/0a0a829c9e3b4637adb426c439c36a27 2024-11-17T01:53:23,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/0a0a829c9e3b4637adb426c439c36a27, entries=150, sequenceid=263, filesize=12.0 K 2024-11-17T01:53:23,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/564a4b288fa449ffa7e90f8834bd925f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/564a4b288fa449ffa7e90f8834bd925f 2024-11-17T01:53:23,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808463764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:23,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/564a4b288fa449ffa7e90f8834bd925f, entries=150, sequenceid=263, filesize=12.0 K 2024-11-17T01:53:23,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for fe092165677ecd92ff68c61a7627feb9 in 170ms, sequenceid=263, compaction requested=true 2024-11-17T01:53:23,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:23,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:53:23,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:23,783 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:53:23,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:53:23,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:23,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:53:23,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T01:53:23,783 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:53:23,784 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150981 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:53:23,784 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/A is initiating minor compaction (all files) 2024-11-17T01:53:23,785 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/A in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:23,785 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/33cf21cd8dfc4067990ba3aca6ec892c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/ee23128a51b445339866926d39cd5413, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/c9b5ee40a07847279edaaa7fa6bb9838, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/88bf822f1e1f480bbee5017bc4c2f35d] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=147.4 K 2024-11-17T01:53:23,785 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:23,785 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/33cf21cd8dfc4067990ba3aca6ec892c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/ee23128a51b445339866926d39cd5413, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/c9b5ee40a07847279edaaa7fa6bb9838, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/88bf822f1e1f480bbee5017bc4c2f35d] 2024-11-17T01:53:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:23,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T01:53:23,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:23,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:23,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:23,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:23,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:23,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:23,788 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33cf21cd8dfc4067990ba3aca6ec892c, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731808399344 2024-11-17T01:53:23,788 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49080 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:53:23,788 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee23128a51b445339866926d39cd5413, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731808401302 2024-11-17T01:53:23,788 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/B is initiating minor compaction (all files) 2024-11-17T01:53:23,788 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/B in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:23,788 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/dfaee356cb38486aaf3c4929c1037469, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/a6513d5187ca428cbb542e10604d8389, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/60d4197e5a6a43789832f9d163b3b6b1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/0a0a829c9e3b4637adb426c439c36a27] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=47.9 K 2024-11-17T01:53:23,788 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9b5ee40a07847279edaaa7fa6bb9838, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731808402031 2024-11-17T01:53:23,789 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting dfaee356cb38486aaf3c4929c1037469, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731808399344 2024-11-17T01:53:23,789 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88bf822f1e1f480bbee5017bc4c2f35d, keycount=250, bloomtype=ROW, size=47.4 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731808402385 2024-11-17T01:53:23,789 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting a6513d5187ca428cbb542e10604d8389, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731808401317 2024-11-17T01:53:23,790 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 60d4197e5a6a43789832f9d163b3b6b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731808402031 2024-11-17T01:53:23,791 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a0a829c9e3b4637adb426c439c36a27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731808402385 2024-11-17T01:53:23,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111767f02acbeeca4afabbbfce1ede687717_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_0/A:col10/1731808403653/Put/seqid=0 2024-11-17T01:53:23,815 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:23,823 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#B#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:23,823 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/b8c1b0d76c7d44e2a5d9063185f96127 is 50, key is test_row_0/B:col10/1731808402385/Put/seqid=0 2024-11-17T01:53:23,831 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117bf0f9c718a974cabba3b415c9a304cec_fe092165677ecd92ff68c61a7627feb9 store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:23,834 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117bf0f9c718a974cabba3b415c9a304cec_fe092165677ecd92ff68c61a7627feb9, store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:23,834 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bf0f9c718a974cabba3b415c9a304cec_fe092165677ecd92ff68c61a7627feb9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:23,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742209_1385 (size=14994) 2024-11-17T01:53:23,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742210_1386 (size=12763) 2024-11-17T01:53:23,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808463816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:23,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808463819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:23,853 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/b8c1b0d76c7d44e2a5d9063185f96127 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/b8c1b0d76c7d44e2a5d9063185f96127 2024-11-17T01:53:23,860 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/B of fe092165677ecd92ff68c61a7627feb9 into b8c1b0d76c7d44e2a5d9063185f96127(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:23,860 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:23,860 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/B, priority=12, startTime=1731808403783; duration=0sec 2024-11-17T01:53:23,861 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:23,861 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:B 2024-11-17T01:53:23,861 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:23,862 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36963 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:23,862 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/C is initiating minor compaction (all files) 2024-11-17T01:53:23,862 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/C in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:23,862 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/db24bb9ac28648f482b25f5f226c0e8a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/583adb4bf2284d64b296b9a28215d9d9, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/564a4b288fa449ffa7e90f8834bd925f] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=36.1 K 2024-11-17T01:53:23,862 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting db24bb9ac28648f482b25f5f226c0e8a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731808401317 2024-11-17T01:53:23,862 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 583adb4bf2284d64b296b9a28215d9d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731808402031 2024-11-17T01:53:23,863 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 564a4b288fa449ffa7e90f8834bd925f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731808402385 2024-11-17T01:53:23,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742211_1387 (size=4469) 2024-11-17T01:53:23,869 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#A#compaction#326 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:23,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808463848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:23,870 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/c0b5b80009f44982b1c82324c5572726 is 175, key is test_row_0/A:col10/1731808402385/Put/seqid=0 2024-11-17T01:53:23,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808463851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:23,886 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#C#compaction#328 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:23,886 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/56b20a14602f4c4899685b60a4020d27 is 50, key is test_row_0/C:col10/1731808402385/Put/seqid=0 2024-11-17T01:53:23,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742212_1388 (size=31717) 2024-11-17T01:53:23,896 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/c0b5b80009f44982b1c82324c5572726 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/c0b5b80009f44982b1c82324c5572726 2024-11-17T01:53:23,900 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/A of fe092165677ecd92ff68c61a7627feb9 into c0b5b80009f44982b1c82324c5572726(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:23,900 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:23,900 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/A, priority=12, startTime=1731808403783; duration=0sec 2024-11-17T01:53:23,900 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:23,900 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:A 2024-11-17T01:53:23,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742213_1389 (size=12763) 2024-11-17T01:53:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T01:53:23,954 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:23,954 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:53:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-17T01:53:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T01:53:23,956 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:53:23,956 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:53:23,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:53:23,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808463954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808463954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:23,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808463974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808463974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:23,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808463984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T01:53:24,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T01:53:24,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:24,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:24,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:24,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:24,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:24,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808464174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:24,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808464174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:24,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808464186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:24,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:24,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808464186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:24,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:24,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:24,243 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:24,246 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111767f02acbeeca4afabbbfce1ede687717_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111767f02acbeeca4afabbbfce1ede687717_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:24,247 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/793cba72ff9d43efb32bc3ffc7b28d20, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:24,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/793cba72ff9d43efb32bc3ffc7b28d20 is 175, key is test_row_0/A:col10/1731808403653/Put/seqid=0 2024-11-17T01:53:24,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742214_1390 (size=39949) 2024-11-17T01:53:24,250 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/793cba72ff9d43efb32bc3ffc7b28d20 2024-11-17T01:53:24,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/53a9422c29004df6abbcd334163326ae is 50, key is test_row_0/B:col10/1731808403653/Put/seqid=0 2024-11-17T01:53:24,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T01:53:24,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:24,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:24,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:24,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:24,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:24,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742215_1391 (size=12301) 2024-11-17T01:53:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:24,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/53a9422c29004df6abbcd334163326ae 2024-11-17T01:53:24,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/3512f640abfe4578947999a13fa7f82a is 50, key is test_row_0/C:col10/1731808403653/Put/seqid=0 2024-11-17T01:53:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T01:53:24,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742216_1392 (size=12301) 2024-11-17T01:53:24,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/3512f640abfe4578947999a13fa7f82a 2024-11-17T01:53:24,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/793cba72ff9d43efb32bc3ffc7b28d20 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/793cba72ff9d43efb32bc3ffc7b28d20 2024-11-17T01:53:24,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/793cba72ff9d43efb32bc3ffc7b28d20, entries=200, sequenceid=281, filesize=39.0 K 2024-11-17T01:53:24,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/53a9422c29004df6abbcd334163326ae as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/53a9422c29004df6abbcd334163326ae 2024-11-17T01:53:24,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/53a9422c29004df6abbcd334163326ae, entries=150, sequenceid=281, filesize=12.0 K 2024-11-17T01:53:24,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/3512f640abfe4578947999a13fa7f82a as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/3512f640abfe4578947999a13fa7f82a 2024-11-17T01:53:24,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/3512f640abfe4578947999a13fa7f82a, entries=150, sequenceid=281, filesize=12.0 K 2024-11-17T01:53:24,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for fe092165677ecd92ff68c61a7627feb9 in 508ms, sequenceid=281, compaction requested=false 2024-11-17T01:53:24,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:24,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:24,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T01:53:24,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:24,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:24,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:24,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:24,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:24,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:24,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d3756efa224541f98c78ca1c1b419287_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_0/A:col10/1731808403814/Put/seqid=0 2024-11-17T01:53:24,318 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/56b20a14602f4c4899685b60a4020d27 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/56b20a14602f4c4899685b60a4020d27 2024-11-17T01:53:24,322 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/C of fe092165677ecd92ff68c61a7627feb9 into 56b20a14602f4c4899685b60a4020d27(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:24,322 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:24,322 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/C, priority=13, startTime=1731808403783; duration=0sec 2024-11-17T01:53:24,322 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:24,322 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:C 2024-11-17T01:53:24,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742217_1393 (size=14994) 2024-11-17T01:53:24,343 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:24,346 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d3756efa224541f98c78ca1c1b419287_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d3756efa224541f98c78ca1c1b419287_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:24,347 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/f99978ddb04c4a22b5645156037fdc99, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:24,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/f99978ddb04c4a22b5645156037fdc99 is 175, key is test_row_0/A:col10/1731808403814/Put/seqid=0 2024-11-17T01:53:24,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742218_1394 (size=39949) 2024-11-17T01:53:24,352 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/f99978ddb04c4a22b5645156037fdc99 2024-11-17T01:53:24,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/84efaa4c28414ff3b5c612816af6bf0f is 50, key is test_row_0/B:col10/1731808403814/Put/seqid=0 2024-11-17T01:53:24,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742219_1395 (size=12301) 2024-11-17T01:53:24,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/84efaa4c28414ff3b5c612816af6bf0f 2024-11-17T01:53:24,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/977cf793e1f0425f9eee56bb5ce3559f is 50, key is test_row_0/C:col10/1731808403814/Put/seqid=0 2024-11-17T01:53:24,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T01:53:24,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:24,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:24,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:24,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:24,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:24,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742220_1396 (size=12301) 2024-11-17T01:53:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:24,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808464475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:24,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808464494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:24,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:24,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808464494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:24,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:24,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:24,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808464503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:24,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:24,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808464504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:24,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:24,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T01:53:24,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:24,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:24,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:24,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:24,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T01:53:24,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808464594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:24,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T01:53:24,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:24,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:24,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:24,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:24,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:24,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808464804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:24,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/977cf793e1f0425f9eee56bb5ce3559f 2024-11-17T01:53:24,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/f99978ddb04c4a22b5645156037fdc99 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/f99978ddb04c4a22b5645156037fdc99 2024-11-17T01:53:24,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/f99978ddb04c4a22b5645156037fdc99, entries=200, sequenceid=298, filesize=39.0 K 2024-11-17T01:53:24,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/84efaa4c28414ff3b5c612816af6bf0f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/84efaa4c28414ff3b5c612816af6bf0f 2024-11-17T01:53:24,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/84efaa4c28414ff3b5c612816af6bf0f, entries=150, sequenceid=298, filesize=12.0 K 2024-11-17T01:53:24,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/977cf793e1f0425f9eee56bb5ce3559f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/977cf793e1f0425f9eee56bb5ce3559f 2024-11-17T01:53:24,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/977cf793e1f0425f9eee56bb5ce3559f, entries=150, sequenceid=298, filesize=12.0 K 2024-11-17T01:53:24,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for fe092165677ecd92ff68c61a7627feb9 in 537ms, sequenceid=298, compaction requested=true 2024-11-17T01:53:24,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:24,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:53:24,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:24,838 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:24,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:53:24,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:24,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:53:24,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T01:53:24,838 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:24,839 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:24,839 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/B is initiating minor compaction (all files) 2024-11-17T01:53:24,839 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/B in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:24,839 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/b8c1b0d76c7d44e2a5d9063185f96127, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/53a9422c29004df6abbcd334163326ae, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/84efaa4c28414ff3b5c612816af6bf0f] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=36.5 K 2024-11-17T01:53:24,839 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111615 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:24,839 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/A is initiating minor compaction (all files) 2024-11-17T01:53:24,839 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/A in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:24,839 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/c0b5b80009f44982b1c82324c5572726, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/793cba72ff9d43efb32bc3ffc7b28d20, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/f99978ddb04c4a22b5645156037fdc99] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=109.0 K 2024-11-17T01:53:24,839 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:24,840 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/c0b5b80009f44982b1c82324c5572726, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/793cba72ff9d43efb32bc3ffc7b28d20, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/f99978ddb04c4a22b5645156037fdc99] 2024-11-17T01:53:24,840 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8c1b0d76c7d44e2a5d9063185f96127, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731808402385 2024-11-17T01:53:24,840 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting c0b5b80009f44982b1c82324c5572726, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731808402385 2024-11-17T01:53:24,840 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53a9422c29004df6abbcd334163326ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731808403653 2024-11-17T01:53:24,840 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84efaa4c28414ff3b5c612816af6bf0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731808403814 2024-11-17T01:53:24,840 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 793cba72ff9d43efb32bc3ffc7b28d20, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731808403650 2024-11-17T01:53:24,841 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting f99978ddb04c4a22b5645156037fdc99, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731808403814 2024-11-17T01:53:24,847 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:24,853 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117d1f4eb2453574a2c9bf60d317533fb40_fe092165677ecd92ff68c61a7627feb9 store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:24,856 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117d1f4eb2453574a2c9bf60d317533fb40_fe092165677ecd92ff68c61a7627feb9, store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:24,856 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d1f4eb2453574a2c9bf60d317533fb40_fe092165677ecd92ff68c61a7627feb9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:24,859 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#B#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:24,860 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/4371042a7c80407688a741db07e5f2f7 is 50, key is test_row_0/B:col10/1731808403814/Put/seqid=0 2024-11-17T01:53:24,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742221_1397 (size=4469) 2024-11-17T01:53:24,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742222_1398 (size=12915) 2024-11-17T01:53:24,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T01:53:24,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:24,873 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T01:53:24,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:24,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:24,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:24,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:24,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:24,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:24,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117961d0ef338cc40408d03f695e9f8c13f_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_0/A:col10/1731808404456/Put/seqid=0 2024-11-17T01:53:24,883 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/4371042a7c80407688a741db07e5f2f7 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/4371042a7c80407688a741db07e5f2f7 2024-11-17T01:53:24,896 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/B of fe092165677ecd92ff68c61a7627feb9 into 4371042a7c80407688a741db07e5f2f7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:24,896 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:24,896 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/B, priority=13, startTime=1731808404838; duration=0sec 2024-11-17T01:53:24,897 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:24,897 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:B 2024-11-17T01:53:24,897 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:24,898 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:24,898 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/C is initiating minor compaction (all files) 2024-11-17T01:53:24,898 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/C in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:24,898 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/56b20a14602f4c4899685b60a4020d27, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/3512f640abfe4578947999a13fa7f82a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/977cf793e1f0425f9eee56bb5ce3559f] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=36.5 K 2024-11-17T01:53:24,898 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56b20a14602f4c4899685b60a4020d27, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731808402385 2024-11-17T01:53:24,898 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3512f640abfe4578947999a13fa7f82a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731808403653 2024-11-17T01:53:24,899 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 977cf793e1f0425f9eee56bb5ce3559f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731808403814 2024-11-17T01:53:24,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742223_1399 (size=12454) 2024-11-17T01:53:24,908 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#C#compaction#337 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:24,908 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/e48ef646ac8c4d5ea2dace5751061cad is 50, key is test_row_0/C:col10/1731808403814/Put/seqid=0 2024-11-17T01:53:24,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742224_1400 (size=12915) 2024-11-17T01:53:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:25,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:25,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:25,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808465037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:25,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:25,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808465044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:25,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808465045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:25,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808465046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:25,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:25,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T01:53:25,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808465114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:25,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:25,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808465154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:25,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:25,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:25,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808465154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:25,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:25,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:25,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808465154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:25,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:25,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808465164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:25,273 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#A#compaction#334 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:25,273 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/7b35619ca83849a69da9c1ee5336b542 is 175, key is test_row_0/A:col10/1731808403814/Put/seqid=0 2024-11-17T01:53:25,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742225_1401 (size=31869) 2024-11-17T01:53:25,287 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/7b35619ca83849a69da9c1ee5336b542 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/7b35619ca83849a69da9c1ee5336b542 2024-11-17T01:53:25,291 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/A of fe092165677ecd92ff68c61a7627feb9 into 7b35619ca83849a69da9c1ee5336b542(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:25,291 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:25,291 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/A, priority=13, startTime=1731808404838; duration=0sec 2024-11-17T01:53:25,291 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:25,291 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:A 2024-11-17T01:53:25,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:25,309 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117961d0ef338cc40408d03f695e9f8c13f_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117961d0ef338cc40408d03f695e9f8c13f_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:25,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/eca51224ef914b428283024d725efd1e, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:25,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/eca51224ef914b428283024d725efd1e is 175, key is test_row_0/A:col10/1731808404456/Put/seqid=0 2024-11-17T01:53:25,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742226_1402 (size=31255) 2024-11-17T01:53:25,320 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/e48ef646ac8c4d5ea2dace5751061cad as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/e48ef646ac8c4d5ea2dace5751061cad 2024-11-17T01:53:25,330 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/C of fe092165677ecd92ff68c61a7627feb9 into e48ef646ac8c4d5ea2dace5751061cad(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:25,330 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:25,330 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/C, priority=13, startTime=1731808404838; duration=0sec 2024-11-17T01:53:25,330 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:25,330 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:C 2024-11-17T01:53:25,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808465374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:25,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808465374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:25,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:25,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:25,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808465375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:25,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:25,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:25,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808465375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:25,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:25,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808465634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:25,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808465684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:25,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808465685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:25,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:25,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808465685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:25,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:25,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:25,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808465685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:25,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:25,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:25,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:25,721 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/eca51224ef914b428283024d725efd1e 2024-11-17T01:53:25,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/3d3ec9cbb7544159870275c8616aac40 is 50, key is test_row_0/B:col10/1731808404456/Put/seqid=0 2024-11-17T01:53:25,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742227_1403 (size=12301) 2024-11-17T01:53:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T01:53:26,133 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/3d3ec9cbb7544159870275c8616aac40 2024-11-17T01:53:26,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/8eec671100c44f94ae746a2efef45fc4 is 50, key is test_row_0/C:col10/1731808404456/Put/seqid=0 2024-11-17T01:53:26,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742228_1404 (size=12301) 2024-11-17T01:53:26,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808466194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:26,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808466204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:26,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808466204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:26,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808466204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:26,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:26,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:26,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:26,544 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/8eec671100c44f94ae746a2efef45fc4 2024-11-17T01:53:26,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/eca51224ef914b428283024d725efd1e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/eca51224ef914b428283024d725efd1e 2024-11-17T01:53:26,550 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/eca51224ef914b428283024d725efd1e, entries=150, sequenceid=316, filesize=30.5 K 2024-11-17T01:53:26,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/3d3ec9cbb7544159870275c8616aac40 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/3d3ec9cbb7544159870275c8616aac40 2024-11-17T01:53:26,553 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/3d3ec9cbb7544159870275c8616aac40, entries=150, sequenceid=316, filesize=12.0 K 2024-11-17T01:53:26,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/8eec671100c44f94ae746a2efef45fc4 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/8eec671100c44f94ae746a2efef45fc4 2024-11-17T01:53:26,558 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/8eec671100c44f94ae746a2efef45fc4, entries=150, sequenceid=316, filesize=12.0 K 2024-11-17T01:53:26,558 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for fe092165677ecd92ff68c61a7627feb9 in 1685ms, sequenceid=316, compaction requested=false 2024-11-17T01:53:26,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:26,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:26,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-17T01:53:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-17T01:53:26,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-17T01:53:26,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6050 sec 2024-11-17T01:53:26,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.6090 sec 2024-11-17T01:53:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:26,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T01:53:26,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:26,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:26,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:26,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:26,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:26,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:26,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117950b16311baf405a8e041241f1522910_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_0/A:col10/1731808405032/Put/seqid=0 2024-11-17T01:53:26,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742229_1405 (size=14994) 2024-11-17T01:53:26,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808466724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:26,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808466844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:26,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:26,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:26,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:27,066 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:27,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808467064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:27,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:27,110 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117950b16311baf405a8e041241f1522910_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117950b16311baf405a8e041241f1522910_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:27,111 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/a4dbcd29112e419eacc9419bbf00ae56, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:27,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/a4dbcd29112e419eacc9419bbf00ae56 is 175, key is test_row_0/A:col10/1731808405032/Put/seqid=0 2024-11-17T01:53:27,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742230_1406 (size=39949) 2024-11-17T01:53:27,116 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/a4dbcd29112e419eacc9419bbf00ae56 2024-11-17T01:53:27,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/8083226c186d470ea3d70d4be49bbcec is 50, key is test_row_0/B:col10/1731808405032/Put/seqid=0 2024-11-17T01:53:27,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742231_1407 (size=12301) 2024-11-17T01:53:27,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/8083226c186d470ea3d70d4be49bbcec 2024-11-17T01:53:27,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/9050df42a1024506b9c68fb97868b8ad is 50, key is test_row_0/C:col10/1731808405032/Put/seqid=0 2024-11-17T01:53:27,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742232_1408 (size=12301) 2024-11-17T01:53:27,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808467204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:27,216 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in fe092165677ecd92ff68c61a7627feb9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:53:27,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:27,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:27,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:27,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808467214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:27,220 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in fe092165677ecd92ff68c61a7627feb9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:53:27,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:27,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:27,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:27,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808467224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:27,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808467224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:27,232 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_2' in fe092165677ecd92ff68c61a7627feb9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:53:27,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:27,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:27,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:27,233 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_2' in fe092165677ecd92ff68c61a7627feb9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:53:27,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:27,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:27,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:27,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808467384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:27,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:27,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:27,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:27,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/9050df42a1024506b9c68fb97868b8ad 2024-11-17T01:53:27,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/a4dbcd29112e419eacc9419bbf00ae56 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/a4dbcd29112e419eacc9419bbf00ae56 2024-11-17T01:53:27,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/a4dbcd29112e419eacc9419bbf00ae56, entries=200, sequenceid=335, filesize=39.0 K 2024-11-17T01:53:27,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/8083226c186d470ea3d70d4be49bbcec as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/8083226c186d470ea3d70d4be49bbcec 2024-11-17T01:53:27,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/8083226c186d470ea3d70d4be49bbcec, entries=150, sequenceid=335, filesize=12.0 K 2024-11-17T01:53:27,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/9050df42a1024506b9c68fb97868b8ad as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/9050df42a1024506b9c68fb97868b8ad 2024-11-17T01:53:27,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/9050df42a1024506b9c68fb97868b8ad, entries=150, sequenceid=335, filesize=12.0 K 2024-11-17T01:53:27,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fe092165677ecd92ff68c61a7627feb9 in 930ms, sequenceid=335, compaction requested=true 2024-11-17T01:53:27,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:27,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:53:27,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:27,577 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:27,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:53:27,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:27,577 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:27,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:53:27,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:27,578 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:27,578 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:27,578 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/B is initiating minor compaction (all files) 2024-11-17T01:53:27,578 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/A is initiating minor compaction (all files) 2024-11-17T01:53:27,578 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/B in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:27,578 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/A in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:27,578 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/4371042a7c80407688a741db07e5f2f7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/3d3ec9cbb7544159870275c8616aac40, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/8083226c186d470ea3d70d4be49bbcec] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=36.6 K 2024-11-17T01:53:27,578 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/7b35619ca83849a69da9c1ee5336b542, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/eca51224ef914b428283024d725efd1e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/a4dbcd29112e419eacc9419bbf00ae56] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=100.7 K 2024-11-17T01:53:27,578 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:27,578 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/7b35619ca83849a69da9c1ee5336b542, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/eca51224ef914b428283024d725efd1e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/a4dbcd29112e419eacc9419bbf00ae56] 2024-11-17T01:53:27,579 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 4371042a7c80407688a741db07e5f2f7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731808403814 2024-11-17T01:53:27,579 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b35619ca83849a69da9c1ee5336b542, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731808403814 2024-11-17T01:53:27,581 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting eca51224ef914b428283024d725efd1e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731808404456 2024-11-17T01:53:27,581 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d3ec9cbb7544159870275c8616aac40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731808404456 2024-11-17T01:53:27,581 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4dbcd29112e419eacc9419bbf00ae56, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731808405032 2024-11-17T01:53:27,582 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 8083226c186d470ea3d70d4be49bbcec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731808405032 2024-11-17T01:53:27,589 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:27,592 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#B#compaction#344 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:27,592 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/271c4e1e282f409d85c1580137e7d15d is 50, key is test_row_0/B:col10/1731808405032/Put/seqid=0 2024-11-17T01:53:27,594 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411171acfa162864f4fec884593b4b48607c4_fe092165677ecd92ff68c61a7627feb9 store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:27,596 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411171acfa162864f4fec884593b4b48607c4_fe092165677ecd92ff68c61a7627feb9, store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:27,598 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171acfa162864f4fec884593b4b48607c4_fe092165677ecd92ff68c61a7627feb9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:27,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742233_1409 (size=13017) 2024-11-17T01:53:27,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742234_1410 (size=4469) 2024-11-17T01:53:27,626 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#A#compaction#343 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:27,626 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/c49dcd4b1b884c9494c964359d3d331a is 175, key is test_row_0/A:col10/1731808405032/Put/seqid=0 2024-11-17T01:53:27,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742235_1411 (size=31971) 2024-11-17T01:53:27,641 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/c49dcd4b1b884c9494c964359d3d331a as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/c49dcd4b1b884c9494c964359d3d331a 2024-11-17T01:53:27,645 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/A of fe092165677ecd92ff68c61a7627feb9 into c49dcd4b1b884c9494c964359d3d331a(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:27,645 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:27,645 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/A, priority=13, startTime=1731808407577; duration=0sec 2024-11-17T01:53:27,645 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:27,645 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:A 2024-11-17T01:53:27,645 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:27,646 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:27,646 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/C is initiating minor compaction (all files) 2024-11-17T01:53:27,646 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/C in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:27,646 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/e48ef646ac8c4d5ea2dace5751061cad, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/8eec671100c44f94ae746a2efef45fc4, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/9050df42a1024506b9c68fb97868b8ad] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=36.6 K 2024-11-17T01:53:27,646 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting e48ef646ac8c4d5ea2dace5751061cad, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731808403814 2024-11-17T01:53:27,647 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8eec671100c44f94ae746a2efef45fc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731808404456 2024-11-17T01:53:27,647 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9050df42a1024506b9c68fb97868b8ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731808405032 2024-11-17T01:53:27,652 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#C#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:27,653 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/2e90dd08305149b0a412d3f0ea06160c is 50, key is test_row_0/C:col10/1731808405032/Put/seqid=0 2024-11-17T01:53:27,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742236_1412 (size=13017) 2024-11-17T01:53:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:27,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T01:53:27,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:27,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:27,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:27,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:27,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:27,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:27,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111773fc29e2a2464b219646cb651bfc8035_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_0/A:col10/1731808406722/Put/seqid=0 2024-11-17T01:53:27,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742237_1413 (size=14994) 2024-11-17T01:53:28,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808468000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:28,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:28,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:28,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:28,018 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/271c4e1e282f409d85c1580137e7d15d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/271c4e1e282f409d85c1580137e7d15d 2024-11-17T01:53:28,023 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/B of fe092165677ecd92ff68c61a7627feb9 into 271c4e1e282f409d85c1580137e7d15d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:28,024 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:28,024 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/B, priority=13, startTime=1731808407577; duration=0sec 2024-11-17T01:53:28,024 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:28,024 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:B 2024-11-17T01:53:28,063 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/2e90dd08305149b0a412d3f0ea06160c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/2e90dd08305149b0a412d3f0ea06160c 2024-11-17T01:53:28,067 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/C of fe092165677ecd92ff68c61a7627feb9 into 2e90dd08305149b0a412d3f0ea06160c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:28,067 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:28,067 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/C, priority=13, startTime=1731808407577; duration=0sec 2024-11-17T01:53:28,067 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:28,067 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:C 2024-11-17T01:53:28,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808468114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:28,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:28,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:28,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T01:53:28,124 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:28,125 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:53:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-17T01:53:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T01:53:28,127 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:53:28,128 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:53:28,128 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:53:28,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T01:53:28,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T01:53:28,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:28,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:28,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:28,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:28,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:28,325 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:28,328 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111773fc29e2a2464b219646cb651bfc8035_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111773fc29e2a2464b219646cb651bfc8035_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:28,329 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/5223f472189e4f18b810f4b1d59524cc, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:28,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/5223f472189e4f18b810f4b1d59524cc is 175, key is test_row_0/A:col10/1731808406722/Put/seqid=0 2024-11-17T01:53:28,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808468324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:28,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742238_1414 (size=39949) 2024-11-17T01:53:28,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T01:53:28,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:28,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:28,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:28,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:28,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T01:53:28,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T01:53:28,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:28,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:28,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:28,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:28,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:28,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808468644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:28,736 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/5223f472189e4f18b810f4b1d59524cc 2024-11-17T01:53:28,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T01:53:28,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:28,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:28,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:28,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:28,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:28,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/6035fba51bd34b418886d05926149681 is 50, key is test_row_0/B:col10/1731808406722/Put/seqid=0 2024-11-17T01:53:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T01:53:28,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742239_1415 (size=12301) 2024-11-17T01:53:28,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T01:53:28,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:28,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:28,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:28,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:28,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:28,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:29,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T01:53:29,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:29,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:29,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:29,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:29,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:29,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808469154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:29,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:29,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:29,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:29,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/6035fba51bd34b418886d05926149681 2024-11-17T01:53:29,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/a1174cdfc8934023996c66629e844ee2 is 50, key is test_row_0/C:col10/1731808406722/Put/seqid=0 2024-11-17T01:53:29,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T01:53:29,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:29,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:29,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:29,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:29,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:29,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742240_1416 (size=12301) 2024-11-17T01:53:29,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808469235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:29,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808469236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:29,237 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in fe092165677ecd92ff68c61a7627feb9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:53:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:29,239 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in fe092165677ecd92ff68c61a7627feb9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:53:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:29,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808469244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:29,249 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_2' in fe092165677ecd92ff68c61a7627feb9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:53:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:29,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808469254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:29,259 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_2' in fe092165677ecd92ff68c61a7627feb9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:53:29,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:29,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:29,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T01:53:29,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T01:53:29,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:29,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:29,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:29,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:29,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:29,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T01:53:29,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:29,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:29,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:29,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:29,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:29,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/a1174cdfc8934023996c66629e844ee2 2024-11-17T01:53:29,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/5223f472189e4f18b810f4b1d59524cc as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/5223f472189e4f18b810f4b1d59524cc 2024-11-17T01:53:29,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/5223f472189e4f18b810f4b1d59524cc, entries=200, sequenceid=353, filesize=39.0 K 2024-11-17T01:53:29,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/6035fba51bd34b418886d05926149681 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/6035fba51bd34b418886d05926149681 2024-11-17T01:53:29,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/6035fba51bd34b418886d05926149681, entries=150, sequenceid=353, filesize=12.0 K 2024-11-17T01:53:29,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/a1174cdfc8934023996c66629e844ee2 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/a1174cdfc8934023996c66629e844ee2 2024-11-17T01:53:29,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/a1174cdfc8934023996c66629e844ee2, entries=150, sequenceid=353, filesize=12.0 K 2024-11-17T01:53:29,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for fe092165677ecd92ff68c61a7627feb9 in 1708ms, sequenceid=353, compaction requested=false 2024-11-17T01:53:29,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:29,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T01:53:29,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:29,655 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T01:53:29,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:29,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:29,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:29,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:29,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:29,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:29,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170af3bf3b5a364d559224f3af04b3616e_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_0/A:col10/1731808407979/Put/seqid=0 2024-11-17T01:53:29,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742241_1417 (size=12454) 2024-11-17T01:53:30,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:30,069 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170af3bf3b5a364d559224f3af04b3616e_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170af3bf3b5a364d559224f3af04b3616e_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:30,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/ffc3832544924c7fb8b6e4bee13362bb, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:30,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/ffc3832544924c7fb8b6e4bee13362bb is 175, key is test_row_0/A:col10/1731808407979/Put/seqid=0 2024-11-17T01:53:30,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742242_1418 (size=31255) 2024-11-17T01:53:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:30,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T01:53:30,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808470277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:30,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:30,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808470384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:30,476 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/ffc3832544924c7fb8b6e4bee13362bb 2024-11-17T01:53:30,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/f9612e04f2a5457cbb1280d6de13a947 is 50, key is test_row_0/B:col10/1731808407979/Put/seqid=0 2024-11-17T01:53:30,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742243_1419 (size=12301) 2024-11-17T01:53:30,502 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T01:53:30,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808470604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:30,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:30,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:30,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:30,884 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/f9612e04f2a5457cbb1280d6de13a947 2024-11-17T01:53:30,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/97d16cf709dd4c928ece4366ebb8d83d is 50, key is test_row_0/C:col10/1731808407979/Put/seqid=0 2024-11-17T01:53:30,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742244_1420 (size=12301) 2024-11-17T01:53:30,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:30,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808470915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:30,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:30,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:30,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:31,292 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/97d16cf709dd4c928ece4366ebb8d83d 2024-11-17T01:53:31,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/ffc3832544924c7fb8b6e4bee13362bb as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/ffc3832544924c7fb8b6e4bee13362bb 2024-11-17T01:53:31,299 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/ffc3832544924c7fb8b6e4bee13362bb, entries=150, sequenceid=370, filesize=30.5 K 2024-11-17T01:53:31,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/f9612e04f2a5457cbb1280d6de13a947 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/f9612e04f2a5457cbb1280d6de13a947 2024-11-17T01:53:31,303 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/f9612e04f2a5457cbb1280d6de13a947, entries=150, sequenceid=370, filesize=12.0 K 2024-11-17T01:53:31,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/97d16cf709dd4c928ece4366ebb8d83d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/97d16cf709dd4c928ece4366ebb8d83d 2024-11-17T01:53:31,307 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/97d16cf709dd4c928ece4366ebb8d83d, entries=150, sequenceid=370, filesize=12.0 K 2024-11-17T01:53:31,308 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for fe092165677ecd92ff68c61a7627feb9 in 1653ms, sequenceid=370, compaction requested=true 2024-11-17T01:53:31,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:31,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:31,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-17T01:53:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-17T01:53:31,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-17T01:53:31,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1810 sec 2024-11-17T01:53:31,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 3.1860 sec 2024-11-17T01:53:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:31,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T01:53:31,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:31,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:31,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:31,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:31,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:31,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:31,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bfd7e14cd94847068039796fba6cf3b8_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_0/A:col10/1731808410275/Put/seqid=0 2024-11-17T01:53:31,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742245_1421 (size=14994) 2024-11-17T01:53:31,447 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:31,450 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bfd7e14cd94847068039796fba6cf3b8_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bfd7e14cd94847068039796fba6cf3b8_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:31,451 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/78736ac7326b4d12b52607689f46df22, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:31,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/78736ac7326b4d12b52607689f46df22 is 175, key is test_row_0/A:col10/1731808410275/Put/seqid=0 2024-11-17T01:53:31,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742246_1422 (size=39949) 2024-11-17T01:53:31,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808471508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:31,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:31,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:31,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:31,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808471624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:31,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:31,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:31,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:31,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:31,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808471834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:31,861 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=388, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/78736ac7326b4d12b52607689f46df22 2024-11-17T01:53:31,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/99b7a10012a443e4af24ed38bb48a445 is 50, key is test_row_0/B:col10/1731808410275/Put/seqid=0 2024-11-17T01:53:31,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742247_1423 (size=12301) 2024-11-17T01:53:31,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/99b7a10012a443e4af24ed38bb48a445 2024-11-17T01:53:31,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/e5dfeb693e394f48bd04c6161c4c6d90 is 50, key is test_row_0/C:col10/1731808410275/Put/seqid=0 2024-11-17T01:53:31,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742248_1424 (size=12301) 2024-11-17T01:53:32,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:32,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808472144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:32,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:32,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:32,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:32,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/e5dfeb693e394f48bd04c6161c4c6d90 2024-11-17T01:53:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T01:53:32,284 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:32,285 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:53:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-17T01:53:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T01:53:32,287 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:53:32,288 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:53:32,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:53:32,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/78736ac7326b4d12b52607689f46df22 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/78736ac7326b4d12b52607689f46df22 2024-11-17T01:53:32,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/78736ac7326b4d12b52607689f46df22, entries=200, sequenceid=388, filesize=39.0 K 2024-11-17T01:53:32,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/99b7a10012a443e4af24ed38bb48a445 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/99b7a10012a443e4af24ed38bb48a445 2024-11-17T01:53:32,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/99b7a10012a443e4af24ed38bb48a445, entries=150, sequenceid=388, filesize=12.0 K 2024-11-17T01:53:32,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/e5dfeb693e394f48bd04c6161c4c6d90 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/e5dfeb693e394f48bd04c6161c4c6d90 2024-11-17T01:53:32,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/e5dfeb693e394f48bd04c6161c4c6d90, entries=150, sequenceid=388, filesize=12.0 K 2024-11-17T01:53:32,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for fe092165677ecd92ff68c61a7627feb9 in 862ms, sequenceid=388, compaction requested=true 2024-11-17T01:53:32,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:32,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:53:32,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:32,300 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:53:32,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:53:32,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:32,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:53:32,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T01:53:32,300 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:53:32,301 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:53:32,301 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/A is initiating minor compaction (all files) 2024-11-17T01:53:32,301 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/A in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:32,301 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/c49dcd4b1b884c9494c964359d3d331a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/5223f472189e4f18b810f4b1d59524cc, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/ffc3832544924c7fb8b6e4bee13362bb, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/78736ac7326b4d12b52607689f46df22] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=139.8 K 2024-11-17T01:53:32,301 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:32,301 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/c49dcd4b1b884c9494c964359d3d331a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/5223f472189e4f18b810f4b1d59524cc, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/ffc3832544924c7fb8b6e4bee13362bb, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/78736ac7326b4d12b52607689f46df22] 2024-11-17T01:53:32,307 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:53:32,307 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting c49dcd4b1b884c9494c964359d3d331a, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731808405032 2024-11-17T01:53:32,307 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/B is initiating minor compaction (all files) 2024-11-17T01:53:32,307 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/B in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:32,307 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/271c4e1e282f409d85c1580137e7d15d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/6035fba51bd34b418886d05926149681, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/f9612e04f2a5457cbb1280d6de13a947, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/99b7a10012a443e4af24ed38bb48a445] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=48.8 K 2024-11-17T01:53:32,307 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5223f472189e4f18b810f4b1d59524cc, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731808406699 2024-11-17T01:53:32,307 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 271c4e1e282f409d85c1580137e7d15d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731808405032 2024-11-17T01:53:32,307 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffc3832544924c7fb8b6e4bee13362bb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731808407979 2024-11-17T01:53:32,307 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 6035fba51bd34b418886d05926149681, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731808406699 2024-11-17T01:53:32,308 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78736ac7326b4d12b52607689f46df22, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731808410263 2024-11-17T01:53:32,308 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting f9612e04f2a5457cbb1280d6de13a947, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731808407979 2024-11-17T01:53:32,308 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 99b7a10012a443e4af24ed38bb48a445, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731808410263 2024-11-17T01:53:32,313 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:32,314 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#B#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:32,314 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411173b993bc9411d46aba9af122c500a3283_fe092165677ecd92ff68c61a7627feb9 store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:32,315 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/8375e571153646ce9c99ff9c05bd4dca is 50, key is test_row_0/B:col10/1731808410275/Put/seqid=0 2024-11-17T01:53:32,317 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411173b993bc9411d46aba9af122c500a3283_fe092165677ecd92ff68c61a7627feb9, store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:32,318 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173b993bc9411d46aba9af122c500a3283_fe092165677ecd92ff68c61a7627feb9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:32,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742249_1425 (size=13153) 2024-11-17T01:53:32,333 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/8375e571153646ce9c99ff9c05bd4dca as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/8375e571153646ce9c99ff9c05bd4dca 2024-11-17T01:53:32,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742250_1426 (size=4469) 2024-11-17T01:53:32,338 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/B of fe092165677ecd92ff68c61a7627feb9 into 8375e571153646ce9c99ff9c05bd4dca(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:32,338 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:32,338 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/B, priority=12, startTime=1731808412300; duration=0sec 2024-11-17T01:53:32,338 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:32,338 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:B 2024-11-17T01:53:32,338 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:53:32,339 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:53:32,339 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/C is initiating minor compaction (all files) 2024-11-17T01:53:32,339 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/C in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:32,339 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/2e90dd08305149b0a412d3f0ea06160c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/a1174cdfc8934023996c66629e844ee2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/97d16cf709dd4c928ece4366ebb8d83d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/e5dfeb693e394f48bd04c6161c4c6d90] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=48.8 K 2024-11-17T01:53:32,340 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e90dd08305149b0a412d3f0ea06160c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731808405032 2024-11-17T01:53:32,340 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting a1174cdfc8934023996c66629e844ee2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731808406699 2024-11-17T01:53:32,340 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 97d16cf709dd4c928ece4366ebb8d83d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731808407979 2024-11-17T01:53:32,340 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting e5dfeb693e394f48bd04c6161c4c6d90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731808410263 2024-11-17T01:53:32,351 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#C#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:32,352 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/3c11ace9c2a74ba8961fadc4dc03f0a0 is 50, key is test_row_0/C:col10/1731808410275/Put/seqid=0 2024-11-17T01:53:32,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742251_1427 (size=13153) 2024-11-17T01:53:32,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T01:53:32,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T01:53:32,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:32,440 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T01:53:32,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:32,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:32,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:32,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:32,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:32,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:32,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174f8a1e0bde98421b993a75974f5c9387_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_0/A:col10/1731808411503/Put/seqid=0 2024-11-17T01:53:32,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742252_1428 (size=12454) 2024-11-17T01:53:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T01:53:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:32,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. as already flushing 2024-11-17T01:53:32,736 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#A#compaction#356 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:32,736 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/35cdfa46292d46f89c098958b2f68d56 is 175, key is test_row_0/A:col10/1731808410275/Put/seqid=0 2024-11-17T01:53:32,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742253_1429 (size=32107) 2024-11-17T01:53:32,764 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/3c11ace9c2a74ba8961fadc4dc03f0a0 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/3c11ace9c2a74ba8961fadc4dc03f0a0 2024-11-17T01:53:32,777 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/C of fe092165677ecd92ff68c61a7627feb9 into 3c11ace9c2a74ba8961fadc4dc03f0a0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:32,777 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:32,777 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/C, priority=12, startTime=1731808412300; duration=0sec 2024-11-17T01:53:32,777 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:32,777 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:C 2024-11-17T01:53:32,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808472799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:32,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:32,857 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174f8a1e0bde98421b993a75974f5c9387_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174f8a1e0bde98421b993a75974f5c9387_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:32,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/614bfe4dc82c41769922497f7de91cbf, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:32,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/614bfe4dc82c41769922497f7de91cbf is 175, key is test_row_0/A:col10/1731808411503/Put/seqid=0 2024-11-17T01:53:32,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742254_1430 (size=31255) 2024-11-17T01:53:32,862 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=403, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/614bfe4dc82c41769922497f7de91cbf 2024-11-17T01:53:32,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/a450a70000854f25a852d83d6a629479 is 50, key is test_row_0/B:col10/1731808411503/Put/seqid=0 2024-11-17T01:53:32,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742255_1431 (size=12301) 2024-11-17T01:53:32,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T01:53:32,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808472924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:32,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:32,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:32,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:33,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:33,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808473134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:33,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:33,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:33,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:33,147 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/35cdfa46292d46f89c098958b2f68d56 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/35cdfa46292d46f89c098958b2f68d56 2024-11-17T01:53:33,151 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/A of fe092165677ecd92ff68c61a7627feb9 into 35cdfa46292d46f89c098958b2f68d56(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:33,151 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:33,151 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/A, priority=12, startTime=1731808412300; duration=0sec 2024-11-17T01:53:33,151 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:33,151 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:A 2024-11-17T01:53:33,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:33,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54322 deadline: 1731808473264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:33,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:33,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54374 deadline: 1731808473264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:33,269 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in fe092165677ecd92ff68c61a7627feb9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:53:33,269 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_2' in fe092165677ecd92ff68c61a7627feb9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:53:33,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:33,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:33,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:33,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:33,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:33,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:33,271 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/a450a70000854f25a852d83d6a629479 2024-11-17T01:53:33,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/606f68a23d9f4a96bbf152cc82b873b1 is 50, key is test_row_0/C:col10/1731808411503/Put/seqid=0 2024-11-17T01:53:33,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:33,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1731808473274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:33,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742256_1432 (size=12301) 2024-11-17T01:53:33,280 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in fe092165677ecd92ff68c61a7627feb9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:53:33,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:33,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:33,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:33,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1731808473284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:33,290 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_2' in fe092165677ecd92ff68c61a7627feb9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:53:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:33,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T01:53:33,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:33,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54380 deadline: 1731808473454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:53:33,577 INFO [Thread-1499 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1499. 2024-11-17T01:53:33,577 DEBUG [Thread-1499 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:53:33,577 DEBUG [Thread-1499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:33,577 DEBUG [Thread-1499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:33,578 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:33,580 INFO [Thread-1501 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1501. 2024-11-17T01:53:33,580 DEBUG [Thread-1501 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:53:33,580 DEBUG [Thread-1501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:33,580 DEBUG [Thread-1501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:33,580 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:33,582 INFO [Thread-1495 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1495. 2024-11-17T01:53:33,582 DEBUG [Thread-1495 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:53:33,582 DEBUG [Thread-1495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:33,582 DEBUG [Thread-1495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:33,582 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:33,587 INFO [Thread-1497 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1497. 2024-11-17T01:53:33,587 DEBUG [Thread-1497 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:53:33,587 DEBUG [Thread-1497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:33,587 DEBUG [Thread-1497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:33,587 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:33,590 INFO [Thread-1493 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1493. 2024-11-17T01:53:33,590 DEBUG [Thread-1493 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:53:33,590 DEBUG [Thread-1493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:33,591 DEBUG [Thread-1493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:33,591 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:33,682 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/606f68a23d9f4a96bbf152cc82b873b1 2024-11-17T01:53:33,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/614bfe4dc82c41769922497f7de91cbf as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/614bfe4dc82c41769922497f7de91cbf 2024-11-17T01:53:33,697 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/614bfe4dc82c41769922497f7de91cbf, entries=150, sequenceid=403, filesize=30.5 K 2024-11-17T01:53:33,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/a450a70000854f25a852d83d6a629479 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/a450a70000854f25a852d83d6a629479 2024-11-17T01:53:33,702 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/a450a70000854f25a852d83d6a629479, entries=150, sequenceid=403, filesize=12.0 K 2024-11-17T01:53:33,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/606f68a23d9f4a96bbf152cc82b873b1 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/606f68a23d9f4a96bbf152cc82b873b1 2024-11-17T01:53:33,708 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/606f68a23d9f4a96bbf152cc82b873b1, entries=150, sequenceid=403, filesize=12.0 K 2024-11-17T01:53:33,709 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for fe092165677ecd92ff68c61a7627feb9 in 1268ms, sequenceid=403, compaction requested=false 2024-11-17T01:53:33,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:33,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:33,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-17T01:53:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-17T01:53:33,712 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-17T01:53:33,712 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4220 sec 2024-11-17T01:53:33,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.4280 sec 2024-11-17T01:53:33,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:33,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T01:53:33,966 INFO [Thread-1484 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1484. 2024-11-17T01:53:33,966 DEBUG [Thread-1484 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:53:33,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:33,966 DEBUG [Thread-1484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:33,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:33,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:33,966 DEBUG [Thread-1484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:33,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:33,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:33,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:33,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:33,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111779941aa5de544d638f61dfa32c6d7967_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_0/A:col10/1731808412796/Put/seqid=0 2024-11-17T01:53:33,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742257_1433 (size=12454) 2024-11-17T01:53:34,378 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:34,388 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111779941aa5de544d638f61dfa32c6d7967_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111779941aa5de544d638f61dfa32c6d7967_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:34,390 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/f8c5bb6bdae5440c96a99b18684743d9, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:34,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/f8c5bb6bdae5440c96a99b18684743d9 is 175, key is test_row_0/A:col10/1731808412796/Put/seqid=0 2024-11-17T01:53:34,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742258_1434 (size=31255) 2024-11-17T01:53:34,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T01:53:34,444 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:34,794 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=424, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/f8c5bb6bdae5440c96a99b18684743d9 2024-11-17T01:53:34,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/90ca393b0729494691dd442ae6f878c3 is 50, key is test_row_0/B:col10/1731808412796/Put/seqid=0 2024-11-17T01:53:34,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742259_1435 (size=12301) 2024-11-17T01:53:35,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/90ca393b0729494691dd442ae6f878c3 2024-11-17T01:53:35,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/a846aa40c56a4ebfb033b9fe5871ea99 is 50, key is test_row_0/C:col10/1731808412796/Put/seqid=0 2024-11-17T01:53:35,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742260_1436 (size=12301) 2024-11-17T01:53:35,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/a846aa40c56a4ebfb033b9fe5871ea99 2024-11-17T01:53:35,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/f8c5bb6bdae5440c96a99b18684743d9 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/f8c5bb6bdae5440c96a99b18684743d9 2024-11-17T01:53:35,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/f8c5bb6bdae5440c96a99b18684743d9, entries=150, sequenceid=424, filesize=30.5 K 2024-11-17T01:53:35,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/90ca393b0729494691dd442ae6f878c3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/90ca393b0729494691dd442ae6f878c3 2024-11-17T01:53:35,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/90ca393b0729494691dd442ae6f878c3, entries=150, sequenceid=424, filesize=12.0 K 2024-11-17T01:53:35,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/a846aa40c56a4ebfb033b9fe5871ea99 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/a846aa40c56a4ebfb033b9fe5871ea99 2024-11-17T01:53:35,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/a846aa40c56a4ebfb033b9fe5871ea99, entries=150, sequenceid=424, filesize=12.0 K 2024-11-17T01:53:35,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for fe092165677ecd92ff68c61a7627feb9 in 1264ms, sequenceid=424, compaction requested=true 2024-11-17T01:53:35,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:35,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:53:35,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:35,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:53:35,231 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:35,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:35,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe092165677ecd92ff68c61a7627feb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:53:35,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:35,231 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:35,231 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:35,231 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94617 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:35,231 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/B is initiating minor compaction (all files) 2024-11-17T01:53:35,231 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/A is initiating minor compaction (all files) 2024-11-17T01:53:35,232 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/B in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:35,232 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/A in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:35,232 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/8375e571153646ce9c99ff9c05bd4dca, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/a450a70000854f25a852d83d6a629479, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/90ca393b0729494691dd442ae6f878c3] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=36.9 K 2024-11-17T01:53:35,232 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/35cdfa46292d46f89c098958b2f68d56, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/614bfe4dc82c41769922497f7de91cbf, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/f8c5bb6bdae5440c96a99b18684743d9] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=92.4 K 2024-11-17T01:53:35,232 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:35,232 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/35cdfa46292d46f89c098958b2f68d56, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/614bfe4dc82c41769922497f7de91cbf, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/f8c5bb6bdae5440c96a99b18684743d9] 2024-11-17T01:53:35,232 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 8375e571153646ce9c99ff9c05bd4dca, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731808410263 2024-11-17T01:53:35,232 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35cdfa46292d46f89c098958b2f68d56, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731808410263 2024-11-17T01:53:35,232 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting a450a70000854f25a852d83d6a629479, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731808411494 2024-11-17T01:53:35,232 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 90ca393b0729494691dd442ae6f878c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731808412753 2024-11-17T01:53:35,232 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 614bfe4dc82c41769922497f7de91cbf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731808411494 2024-11-17T01:53:35,232 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8c5bb6bdae5440c96a99b18684743d9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731808412753 2024-11-17T01:53:35,238 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:35,239 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#B#compaction#364 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:35,239 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/5f62da378d0e4cf1982850389149d980 is 50, key is test_row_0/B:col10/1731808412796/Put/seqid=0 2024-11-17T01:53:35,239 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111710856a14cca5421896db3228dcdc5f0e_fe092165677ecd92ff68c61a7627feb9 store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:35,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742261_1437 (size=13255) 2024-11-17T01:53:35,245 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/121c689a7cad40e19c60664c8ff29e14, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/4f9491068c664ae9a0ea02e0684448f3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/83e6a74b91e046be8fa4c70dce0237e8, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/1454d542dbd5477eb8bf1629d26a04b7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/91130e9b5166416d851fea0271a2a4bf, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/35fc2285351546a7bdbdf54e18656db0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/2c701c8ad7f84cc48be996671ed98d20, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/35c892eaab4541dabc4169cec1064873, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/2fe06fd186b64052bb8b36253c625bf0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/1b3a85bbb20041c492719402b5993115, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/33cf21cd8dfc4067990ba3aca6ec892c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/71f7fe8b593e405cac1a303ca004dbeb, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/ee23128a51b445339866926d39cd5413, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/c9b5ee40a07847279edaaa7fa6bb9838, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/88bf822f1e1f480bbee5017bc4c2f35d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/c0b5b80009f44982b1c82324c5572726, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/793cba72ff9d43efb32bc3ffc7b28d20, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/f99978ddb04c4a22b5645156037fdc99, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/7b35619ca83849a69da9c1ee5336b542, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/eca51224ef914b428283024d725efd1e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/a4dbcd29112e419eacc9419bbf00ae56, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/c49dcd4b1b884c9494c964359d3d331a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/5223f472189e4f18b810f4b1d59524cc, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/ffc3832544924c7fb8b6e4bee13362bb, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/78736ac7326b4d12b52607689f46df22] to archive 2024-11-17T01:53:35,246 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T01:53:35,248 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/5f62da378d0e4cf1982850389149d980 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/5f62da378d0e4cf1982850389149d980 2024-11-17T01:53:35,248 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/121c689a7cad40e19c60664c8ff29e14 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/121c689a7cad40e19c60664c8ff29e14 2024-11-17T01:53:35,250 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111710856a14cca5421896db3228dcdc5f0e_fe092165677ecd92ff68c61a7627feb9, store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:35,250 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111710856a14cca5421896db3228dcdc5f0e_fe092165677ecd92ff68c61a7627feb9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:35,250 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/4f9491068c664ae9a0ea02e0684448f3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/4f9491068c664ae9a0ea02e0684448f3 2024-11-17T01:53:35,252 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/B of fe092165677ecd92ff68c61a7627feb9 into 5f62da378d0e4cf1982850389149d980(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:35,252 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:35,253 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/B, priority=13, startTime=1731808415231; duration=0sec 2024-11-17T01:53:35,253 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:35,253 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:B 2024-11-17T01:53:35,253 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:35,253 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/83e6a74b91e046be8fa4c70dce0237e8 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/83e6a74b91e046be8fa4c70dce0237e8 2024-11-17T01:53:35,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742262_1438 (size=4469) 2024-11-17T01:53:35,254 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:35,255 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): fe092165677ecd92ff68c61a7627feb9/C is initiating minor compaction (all files) 2024-11-17T01:53:35,255 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe092165677ecd92ff68c61a7627feb9/C in TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:35,255 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/3c11ace9c2a74ba8961fadc4dc03f0a0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/606f68a23d9f4a96bbf152cc82b873b1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/a846aa40c56a4ebfb033b9fe5871ea99] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp, totalSize=36.9 K 2024-11-17T01:53:35,256 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/1454d542dbd5477eb8bf1629d26a04b7 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/1454d542dbd5477eb8bf1629d26a04b7 2024-11-17T01:53:35,256 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c11ace9c2a74ba8961fadc4dc03f0a0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731808410263 2024-11-17T01:53:35,256 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#A#compaction#365 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:35,256 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 606f68a23d9f4a96bbf152cc82b873b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731808411494 2024-11-17T01:53:35,256 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting a846aa40c56a4ebfb033b9fe5871ea99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731808412753 2024-11-17T01:53:35,256 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/79a0c0469f1a4d8aaaafe7d92b4c4df2 is 175, key is test_row_0/A:col10/1731808412796/Put/seqid=0 2024-11-17T01:53:35,258 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/91130e9b5166416d851fea0271a2a4bf to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/91130e9b5166416d851fea0271a2a4bf 2024-11-17T01:53:35,259 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/35fc2285351546a7bdbdf54e18656db0 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/35fc2285351546a7bdbdf54e18656db0 2024-11-17T01:53:35,261 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/2c701c8ad7f84cc48be996671ed98d20 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/2c701c8ad7f84cc48be996671ed98d20 2024-11-17T01:53:35,263 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/35c892eaab4541dabc4169cec1064873 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/35c892eaab4541dabc4169cec1064873 2024-11-17T01:53:35,264 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/2fe06fd186b64052bb8b36253c625bf0 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/2fe06fd186b64052bb8b36253c625bf0 2024-11-17T01:53:35,268 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/1b3a85bbb20041c492719402b5993115 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/1b3a85bbb20041c492719402b5993115 2024-11-17T01:53:35,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742263_1439 (size=32209) 2024-11-17T01:53:35,270 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/33cf21cd8dfc4067990ba3aca6ec892c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/33cf21cd8dfc4067990ba3aca6ec892c 2024-11-17T01:53:35,271 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe092165677ecd92ff68c61a7627feb9#C#compaction#366 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:35,271 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/71f7fe8b593e405cac1a303ca004dbeb to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/71f7fe8b593e405cac1a303ca004dbeb 2024-11-17T01:53:35,271 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/f847cf9e3ca84666823c097542d4491c is 50, key is test_row_0/C:col10/1731808412796/Put/seqid=0 2024-11-17T01:53:35,272 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/ee23128a51b445339866926d39cd5413 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/ee23128a51b445339866926d39cd5413 2024-11-17T01:53:35,274 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/c9b5ee40a07847279edaaa7fa6bb9838 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/c9b5ee40a07847279edaaa7fa6bb9838 2024-11-17T01:53:35,275 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/88bf822f1e1f480bbee5017bc4c2f35d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/88bf822f1e1f480bbee5017bc4c2f35d 2024-11-17T01:53:35,276 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/c0b5b80009f44982b1c82324c5572726 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/c0b5b80009f44982b1c82324c5572726 2024-11-17T01:53:35,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742264_1440 (size=13255) 2024-11-17T01:53:35,277 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/793cba72ff9d43efb32bc3ffc7b28d20 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/793cba72ff9d43efb32bc3ffc7b28d20 2024-11-17T01:53:35,278 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/f99978ddb04c4a22b5645156037fdc99 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/f99978ddb04c4a22b5645156037fdc99 2024-11-17T01:53:35,279 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/7b35619ca83849a69da9c1ee5336b542 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/7b35619ca83849a69da9c1ee5336b542 2024-11-17T01:53:35,280 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/eca51224ef914b428283024d725efd1e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/eca51224ef914b428283024d725efd1e 2024-11-17T01:53:35,281 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/a4dbcd29112e419eacc9419bbf00ae56 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/a4dbcd29112e419eacc9419bbf00ae56 2024-11-17T01:53:35,281 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/c49dcd4b1b884c9494c964359d3d331a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/c49dcd4b1b884c9494c964359d3d331a 2024-11-17T01:53:35,282 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/5223f472189e4f18b810f4b1d59524cc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/5223f472189e4f18b810f4b1d59524cc 2024-11-17T01:53:35,283 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/ffc3832544924c7fb8b6e4bee13362bb to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/ffc3832544924c7fb8b6e4bee13362bb 2024-11-17T01:53:35,284 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/78736ac7326b4d12b52607689f46df22 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/78736ac7326b4d12b52607689f46df22 2024-11-17T01:53:35,287 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/e4f4ed3caca2494dac8355798c52bf92, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/314322d603594012be45fe8c0db274e1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/13599ccef142408dbd24493534554406, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/e226291e7dc142ae947a5f6b65fb2007, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/e7440811ba1e416c92bd6943a761245e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/5ad7950f20604d69aff208cb88040a2d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/b1c3f5fb848a426fbbbfc6cd7b6f0a6d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/0f051c7c89b74bdeb1bcc6d0327afa9c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/95e8fabb18b342aabd610722a8467b56, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/c7f7350843b44bb2b96a408100918e29, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/dfaee356cb38486aaf3c4929c1037469, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/ff81058e6be4437fa87411f498980f9d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/a6513d5187ca428cbb542e10604d8389, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/60d4197e5a6a43789832f9d163b3b6b1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/b8c1b0d76c7d44e2a5d9063185f96127, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/0a0a829c9e3b4637adb426c439c36a27, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/53a9422c29004df6abbcd334163326ae, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/4371042a7c80407688a741db07e5f2f7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/84efaa4c28414ff3b5c612816af6bf0f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/3d3ec9cbb7544159870275c8616aac40, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/271c4e1e282f409d85c1580137e7d15d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/8083226c186d470ea3d70d4be49bbcec, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/6035fba51bd34b418886d05926149681, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/f9612e04f2a5457cbb1280d6de13a947, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/8375e571153646ce9c99ff9c05bd4dca, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/99b7a10012a443e4af24ed38bb48a445, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/a450a70000854f25a852d83d6a629479, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/90ca393b0729494691dd442ae6f878c3] to archive 2024-11-17T01:53:35,288 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T01:53:35,289 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/e4f4ed3caca2494dac8355798c52bf92 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/e4f4ed3caca2494dac8355798c52bf92 2024-11-17T01:53:35,290 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/314322d603594012be45fe8c0db274e1 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/314322d603594012be45fe8c0db274e1 2024-11-17T01:53:35,291 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/13599ccef142408dbd24493534554406 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/13599ccef142408dbd24493534554406 2024-11-17T01:53:35,292 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/e226291e7dc142ae947a5f6b65fb2007 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/e226291e7dc142ae947a5f6b65fb2007 2024-11-17T01:53:35,293 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/e7440811ba1e416c92bd6943a761245e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/e7440811ba1e416c92bd6943a761245e 2024-11-17T01:53:35,294 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/5ad7950f20604d69aff208cb88040a2d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/5ad7950f20604d69aff208cb88040a2d 2024-11-17T01:53:35,295 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/b1c3f5fb848a426fbbbfc6cd7b6f0a6d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/b1c3f5fb848a426fbbbfc6cd7b6f0a6d 2024-11-17T01:53:35,296 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/0f051c7c89b74bdeb1bcc6d0327afa9c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/0f051c7c89b74bdeb1bcc6d0327afa9c 2024-11-17T01:53:35,296 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/95e8fabb18b342aabd610722a8467b56 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/95e8fabb18b342aabd610722a8467b56 2024-11-17T01:53:35,297 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/c7f7350843b44bb2b96a408100918e29 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/c7f7350843b44bb2b96a408100918e29 2024-11-17T01:53:35,298 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/dfaee356cb38486aaf3c4929c1037469 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/dfaee356cb38486aaf3c4929c1037469 2024-11-17T01:53:35,299 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/ff81058e6be4437fa87411f498980f9d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/ff81058e6be4437fa87411f498980f9d 2024-11-17T01:53:35,300 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/a6513d5187ca428cbb542e10604d8389 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/a6513d5187ca428cbb542e10604d8389 2024-11-17T01:53:35,301 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/60d4197e5a6a43789832f9d163b3b6b1 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/60d4197e5a6a43789832f9d163b3b6b1 2024-11-17T01:53:35,301 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/b8c1b0d76c7d44e2a5d9063185f96127 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/b8c1b0d76c7d44e2a5d9063185f96127 2024-11-17T01:53:35,302 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/0a0a829c9e3b4637adb426c439c36a27 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/0a0a829c9e3b4637adb426c439c36a27 2024-11-17T01:53:35,303 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/53a9422c29004df6abbcd334163326ae to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/53a9422c29004df6abbcd334163326ae 2024-11-17T01:53:35,304 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/4371042a7c80407688a741db07e5f2f7 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/4371042a7c80407688a741db07e5f2f7 2024-11-17T01:53:35,305 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/84efaa4c28414ff3b5c612816af6bf0f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/84efaa4c28414ff3b5c612816af6bf0f 2024-11-17T01:53:35,305 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/3d3ec9cbb7544159870275c8616aac40 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/3d3ec9cbb7544159870275c8616aac40 2024-11-17T01:53:35,306 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/271c4e1e282f409d85c1580137e7d15d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/271c4e1e282f409d85c1580137e7d15d 2024-11-17T01:53:35,307 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/8083226c186d470ea3d70d4be49bbcec to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/8083226c186d470ea3d70d4be49bbcec 2024-11-17T01:53:35,308 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/6035fba51bd34b418886d05926149681 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/6035fba51bd34b418886d05926149681 2024-11-17T01:53:35,308 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/f9612e04f2a5457cbb1280d6de13a947 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/f9612e04f2a5457cbb1280d6de13a947 2024-11-17T01:53:35,309 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/8375e571153646ce9c99ff9c05bd4dca to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/8375e571153646ce9c99ff9c05bd4dca 2024-11-17T01:53:35,310 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/99b7a10012a443e4af24ed38bb48a445 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/99b7a10012a443e4af24ed38bb48a445 2024-11-17T01:53:35,310 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/a450a70000854f25a852d83d6a629479 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/a450a70000854f25a852d83d6a629479 2024-11-17T01:53:35,311 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/90ca393b0729494691dd442ae6f878c3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/90ca393b0729494691dd442ae6f878c3 2024-11-17T01:53:35,314 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/ca84c2a725994c4a9517c200f98d9ed0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/bc17e78b9d3849a1962cfe510ee71df3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/a0cdde08ea4d41c5bb7b7c185b8a091a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/022ac1a7460a4ca2b7be140fefe2578c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/84c3bc6c7fb5429494ed317a31727e1c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/b0a0f80db2bb49cf96cdf924411b537b, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/dc199d542ec54d71936dd548a6f9e2b0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/976141a2490d46bdb0b16630448b2de1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/50bea256784948848a21686766f5df0a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/cdcc03265ed84b0c8a06e8f7f87c61df, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/2a85c94abf5a498bb4becdf18e1b9255, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/db24bb9ac28648f482b25f5f226c0e8a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/130d9ed4e36e40b8a59580a41f0db6f5, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/583adb4bf2284d64b296b9a28215d9d9, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/56b20a14602f4c4899685b60a4020d27, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/564a4b288fa449ffa7e90f8834bd925f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/3512f640abfe4578947999a13fa7f82a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/e48ef646ac8c4d5ea2dace5751061cad, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/977cf793e1f0425f9eee56bb5ce3559f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/8eec671100c44f94ae746a2efef45fc4, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/2e90dd08305149b0a412d3f0ea06160c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/9050df42a1024506b9c68fb97868b8ad, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/a1174cdfc8934023996c66629e844ee2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/97d16cf709dd4c928ece4366ebb8d83d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/e5dfeb693e394f48bd04c6161c4c6d90] to archive 2024-11-17T01:53:35,315 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T01:53:35,316 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/ca84c2a725994c4a9517c200f98d9ed0 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/ca84c2a725994c4a9517c200f98d9ed0 2024-11-17T01:53:35,317 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/bc17e78b9d3849a1962cfe510ee71df3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/bc17e78b9d3849a1962cfe510ee71df3 2024-11-17T01:53:35,318 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/a0cdde08ea4d41c5bb7b7c185b8a091a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/a0cdde08ea4d41c5bb7b7c185b8a091a 2024-11-17T01:53:35,319 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/022ac1a7460a4ca2b7be140fefe2578c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/022ac1a7460a4ca2b7be140fefe2578c 2024-11-17T01:53:35,320 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/84c3bc6c7fb5429494ed317a31727e1c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/84c3bc6c7fb5429494ed317a31727e1c 2024-11-17T01:53:35,321 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/b0a0f80db2bb49cf96cdf924411b537b to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/b0a0f80db2bb49cf96cdf924411b537b 2024-11-17T01:53:35,322 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/dc199d542ec54d71936dd548a6f9e2b0 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/dc199d542ec54d71936dd548a6f9e2b0 2024-11-17T01:53:35,323 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/976141a2490d46bdb0b16630448b2de1 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/976141a2490d46bdb0b16630448b2de1 2024-11-17T01:53:35,324 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/50bea256784948848a21686766f5df0a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/50bea256784948848a21686766f5df0a 2024-11-17T01:53:35,325 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/cdcc03265ed84b0c8a06e8f7f87c61df to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/cdcc03265ed84b0c8a06e8f7f87c61df 2024-11-17T01:53:35,325 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/2a85c94abf5a498bb4becdf18e1b9255 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/2a85c94abf5a498bb4becdf18e1b9255 2024-11-17T01:53:35,326 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/db24bb9ac28648f482b25f5f226c0e8a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/db24bb9ac28648f482b25f5f226c0e8a 2024-11-17T01:53:35,328 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/130d9ed4e36e40b8a59580a41f0db6f5 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/130d9ed4e36e40b8a59580a41f0db6f5 2024-11-17T01:53:35,329 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/583adb4bf2284d64b296b9a28215d9d9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/583adb4bf2284d64b296b9a28215d9d9 2024-11-17T01:53:35,329 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/56b20a14602f4c4899685b60a4020d27 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/56b20a14602f4c4899685b60a4020d27 2024-11-17T01:53:35,330 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/564a4b288fa449ffa7e90f8834bd925f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/564a4b288fa449ffa7e90f8834bd925f 2024-11-17T01:53:35,331 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/3512f640abfe4578947999a13fa7f82a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/3512f640abfe4578947999a13fa7f82a 2024-11-17T01:53:35,332 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/e48ef646ac8c4d5ea2dace5751061cad to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/e48ef646ac8c4d5ea2dace5751061cad 2024-11-17T01:53:35,333 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/977cf793e1f0425f9eee56bb5ce3559f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/977cf793e1f0425f9eee56bb5ce3559f 2024-11-17T01:53:35,334 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/8eec671100c44f94ae746a2efef45fc4 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/8eec671100c44f94ae746a2efef45fc4 2024-11-17T01:53:35,334 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/2e90dd08305149b0a412d3f0ea06160c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/2e90dd08305149b0a412d3f0ea06160c 2024-11-17T01:53:35,335 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/9050df42a1024506b9c68fb97868b8ad to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/9050df42a1024506b9c68fb97868b8ad 2024-11-17T01:53:35,336 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/a1174cdfc8934023996c66629e844ee2 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/a1174cdfc8934023996c66629e844ee2 2024-11-17T01:53:35,337 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/97d16cf709dd4c928ece4366ebb8d83d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/97d16cf709dd4c928ece4366ebb8d83d 2024-11-17T01:53:35,337 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/e5dfeb693e394f48bd04c6161c4c6d90 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/e5dfeb693e394f48bd04c6161c4c6d90 2024-11-17T01:53:35,673 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/79a0c0469f1a4d8aaaafe7d92b4c4df2 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/79a0c0469f1a4d8aaaafe7d92b4c4df2 2024-11-17T01:53:35,677 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/A of fe092165677ecd92ff68c61a7627feb9 into 79a0c0469f1a4d8aaaafe7d92b4c4df2(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:35,677 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:35,677 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/A, priority=13, startTime=1731808415231; duration=0sec 2024-11-17T01:53:35,678 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:35,678 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:A 2024-11-17T01:53:35,680 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/f847cf9e3ca84666823c097542d4491c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/f847cf9e3ca84666823c097542d4491c 2024-11-17T01:53:35,684 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe092165677ecd92ff68c61a7627feb9/C of fe092165677ecd92ff68c61a7627feb9 into f847cf9e3ca84666823c097542d4491c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:35,684 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe092165677ecd92ff68c61a7627feb9: 2024-11-17T01:53:35,684 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., storeName=fe092165677ecd92ff68c61a7627feb9/C, priority=13, startTime=1731808415231; duration=0sec 2024-11-17T01:53:35,684 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:35,684 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe092165677ecd92ff68c61a7627feb9:C 2024-11-17T01:53:43,296 INFO [Thread-1488 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1488. 2024-11-17T01:53:43,296 DEBUG [Thread-1488 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:53:43,297 DEBUG [Thread-1488 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:43,297 DEBUG [Thread-1488 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:43,297 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:43,316 INFO [Thread-1486 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1486. 2024-11-17T01:53:43,316 DEBUG [Thread-1486 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:53:43,316 DEBUG [Thread-1486 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:43,316 DEBUG [Thread-1486 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:43,316 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:43,326 INFO [Thread-1482 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1482. 2024-11-17T01:53:43,326 DEBUG [Thread-1482 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:53:43,326 DEBUG [Thread-1482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:43,326 DEBUG [Thread-1482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:43,326 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:43,364 INFO [Thread-1490 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1490. 2024-11-17T01:53:43,364 DEBUG [Thread-1490 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:53:43,364 DEBUG [Thread-1490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:43,364 DEBUG [Thread-1490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:43,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:43,364 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T01:53:43,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-17T01:53:43,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 129 2024-11-17T01:53:43,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-17T01:53:43,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-17T01:53:43,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-17T01:53:43,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T01:53:43,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T01:53:43,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 951 2024-11-17T01:53:43,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2850 rows 2024-11-17T01:53:43,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 953 2024-11-17T01:53:43,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2857 rows 2024-11-17T01:53:43,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 946 2024-11-17T01:53:43,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2838 rows 2024-11-17T01:53:43,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 943 2024-11-17T01:53:43,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2821 rows 2024-11-17T01:53:43,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 947 2024-11-17T01:53:43,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2837 rows 2024-11-17T01:53:43,365 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T01:53:43,365 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:53:43,365 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:43,365 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:43,365 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:53:43,366 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T01:53:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T01:53:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T01:53:43,371 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808423370"}]},"ts":"1731808423370"} 2024-11-17T01:53:43,372 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T01:53:43,372 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T01:53:43,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T01:53:43,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe092165677ecd92ff68c61a7627feb9, UNASSIGN}] 2024-11-17T01:53:43,376 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe092165677ecd92ff68c61a7627feb9, UNASSIGN 2024-11-17T01:53:43,377 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=fe092165677ecd92ff68c61a7627feb9, regionState=CLOSING, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:43,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe092165677ecd92ff68c61a7627feb9, UNASSIGN because future has completed 2024-11-17T01:53:43,379 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T01:53:43,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173}] 2024-11-17T01:53:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T01:53:43,531 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:43,531 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T01:53:43,531 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing fe092165677ecd92ff68c61a7627feb9, disabling compactions & flushes 2024-11-17T01:53:43,531 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:43,531 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:43,531 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. after waiting 0 ms 2024-11-17T01:53:43,531 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:43,532 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing fe092165677ecd92ff68c61a7627feb9 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-17T01:53:43,532 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=A 2024-11-17T01:53:43,532 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:43,532 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=B 2024-11-17T01:53:43,532 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:43,532 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe092165677ecd92ff68c61a7627feb9, store=C 2024-11-17T01:53:43,532 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:43,537 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175e99d0f630a242bfaa0b3e234691f0b5_fe092165677ecd92ff68c61a7627feb9 is 50, key is test_row_1/A:col10/1731808423354/Put/seqid=0 2024-11-17T01:53:43,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742265_1441 (size=9914) 2024-11-17T01:53:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T01:53:43,941 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:43,945 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175e99d0f630a242bfaa0b3e234691f0b5_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175e99d0f630a242bfaa0b3e234691f0b5_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:43,946 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/d16af78848dc43c28a0a4bd6baff63e5, store: [table=TestAcidGuarantees family=A region=fe092165677ecd92ff68c61a7627feb9] 2024-11-17T01:53:43,946 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/d16af78848dc43c28a0a4bd6baff63e5 is 175, key is test_row_1/A:col10/1731808423354/Put/seqid=0 2024-11-17T01:53:43,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742266_1442 (size=22561) 2024-11-17T01:53:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T01:53:44,350 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=434, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/d16af78848dc43c28a0a4bd6baff63e5 2024-11-17T01:53:44,358 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/86b9d0e368744adb9d43dd7bb3c18919 is 50, key is test_row_1/B:col10/1731808423354/Put/seqid=0 2024-11-17T01:53:44,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742267_1443 (size=9857) 2024-11-17T01:53:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T01:53:44,771 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/86b9d0e368744adb9d43dd7bb3c18919 2024-11-17T01:53:44,777 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/1901502be46542f28ff8921d5e9f794a is 50, key is test_row_1/C:col10/1731808423354/Put/seqid=0 2024-11-17T01:53:44,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742268_1444 (size=9857) 2024-11-17T01:53:45,181 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/1901502be46542f28ff8921d5e9f794a 2024-11-17T01:53:45,185 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/A/d16af78848dc43c28a0a4bd6baff63e5 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/d16af78848dc43c28a0a4bd6baff63e5 2024-11-17T01:53:45,188 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/d16af78848dc43c28a0a4bd6baff63e5, entries=100, sequenceid=434, filesize=22.0 K 2024-11-17T01:53:45,188 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/B/86b9d0e368744adb9d43dd7bb3c18919 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/86b9d0e368744adb9d43dd7bb3c18919 2024-11-17T01:53:45,191 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/86b9d0e368744adb9d43dd7bb3c18919, entries=100, sequenceid=434, filesize=9.6 K 2024-11-17T01:53:45,192 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/.tmp/C/1901502be46542f28ff8921d5e9f794a as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/1901502be46542f28ff8921d5e9f794a 2024-11-17T01:53:45,195 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/1901502be46542f28ff8921d5e9f794a, entries=100, sequenceid=434, filesize=9.6 K 2024-11-17T01:53:45,195 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for fe092165677ecd92ff68c61a7627feb9 in 1664ms, sequenceid=434, compaction requested=false 2024-11-17T01:53:45,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/35cdfa46292d46f89c098958b2f68d56, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/614bfe4dc82c41769922497f7de91cbf, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/f8c5bb6bdae5440c96a99b18684743d9] to archive 2024-11-17T01:53:45,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T01:53:45,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/35cdfa46292d46f89c098958b2f68d56 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/35cdfa46292d46f89c098958b2f68d56 2024-11-17T01:53:45,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/614bfe4dc82c41769922497f7de91cbf to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/614bfe4dc82c41769922497f7de91cbf 2024-11-17T01:53:45,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/f8c5bb6bdae5440c96a99b18684743d9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/f8c5bb6bdae5440c96a99b18684743d9 2024-11-17T01:53:45,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/3c11ace9c2a74ba8961fadc4dc03f0a0, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/606f68a23d9f4a96bbf152cc82b873b1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/a846aa40c56a4ebfb033b9fe5871ea99] to archive 2024-11-17T01:53:45,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T01:53:45,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/3c11ace9c2a74ba8961fadc4dc03f0a0 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/3c11ace9c2a74ba8961fadc4dc03f0a0 2024-11-17T01:53:45,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/606f68a23d9f4a96bbf152cc82b873b1 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/606f68a23d9f4a96bbf152cc82b873b1 2024-11-17T01:53:45,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/a846aa40c56a4ebfb033b9fe5871ea99 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/a846aa40c56a4ebfb033b9fe5871ea99 2024-11-17T01:53:45,208 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/recovered.edits/437.seqid, newMaxSeqId=437, maxSeqId=4 2024-11-17T01:53:45,208 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. 2024-11-17T01:53:45,208 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for fe092165677ecd92ff68c61a7627feb9: Waiting for close lock at 1731808423531Running coprocessor pre-close hooks at 1731808423531Disabling compacts and flushes for region at 1731808423531Disabling writes for close at 1731808423531Obtaining lock to block concurrent updates at 1731808423532 (+1 ms)Preparing flush snapshotting stores in fe092165677ecd92ff68c61a7627feb9 at 1731808423532Finished memstore snapshotting TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731808423532Flushing stores of TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9. at 1731808423532Flushing fe092165677ecd92ff68c61a7627feb9/A: creating writer at 1731808423532Flushing mob file fe092165677ecd92ff68c61a7627feb9/A: appending metadata at 1731808423537 (+5 ms)Flushing mob file fe092165677ecd92ff68c61a7627feb9/A: closing flushed file at 1731808423537Flushing fe092165677ecd92ff68c61a7627feb9/A: appending metadata at 1731808423946 (+409 ms)Flushing fe092165677ecd92ff68c61a7627feb9/A: closing flushed file at 1731808423946Flushing fe092165677ecd92ff68c61a7627feb9/B: creating writer at 1731808424354 (+408 ms)Flushing fe092165677ecd92ff68c61a7627feb9/B: appending metadata at 1731808424357 (+3 ms)Flushing fe092165677ecd92ff68c61a7627feb9/B: closing flushed file at 1731808424357Flushing fe092165677ecd92ff68c61a7627feb9/C: creating writer at 1731808424773 (+416 ms)Flushing fe092165677ecd92ff68c61a7627feb9/C: appending metadata at 1731808424776 (+3 ms)Flushing fe092165677ecd92ff68c61a7627feb9/C: closing flushed file at 1731808424776Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@573fd83a: reopening flushed file at 1731808425184 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2616b180: reopening flushed file at 1731808425188 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8f44c3b: reopening flushed file at 1731808425191 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for fe092165677ecd92ff68c61a7627feb9 in 1664ms, sequenceid=434, compaction requested=false at 1731808425195 (+4 ms)Writing region close event to WAL at 1731808425205 (+10 ms)Running coprocessor post-close hooks at 1731808425208 (+3 ms)Closed at 1731808425208 2024-11-17T01:53:45,210 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,210 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=fe092165677ecd92ff68c61a7627feb9, regionState=CLOSED 2024-11-17T01:53:45,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 because future has completed 2024-11-17T01:53:45,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-17T01:53:45,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure fe092165677ecd92ff68c61a7627feb9, server=c6c0d7a0a7c0,41157,1731808292173 in 1.8330 sec 2024-11-17T01:53:45,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-17T01:53:45,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe092165677ecd92ff68c61a7627feb9, UNASSIGN in 1.8380 sec 2024-11-17T01:53:45,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-17T01:53:45,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8420 sec 2024-11-17T01:53:45,218 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808425217"}]},"ts":"1731808425217"} 2024-11-17T01:53:45,219 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T01:53:45,219 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T01:53:45,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8530 sec 2024-11-17T01:53:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T01:53:45,515 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:45,515 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T01:53:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:53:45,516 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:53:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T01:53:45,517 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:53:45,519 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,521 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A, FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B, FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C, FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/recovered.edits] 2024-11-17T01:53:45,522 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/79a0c0469f1a4d8aaaafe7d92b4c4df2 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/79a0c0469f1a4d8aaaafe7d92b4c4df2 2024-11-17T01:53:45,523 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/d16af78848dc43c28a0a4bd6baff63e5 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/A/d16af78848dc43c28a0a4bd6baff63e5 2024-11-17T01:53:45,525 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/5f62da378d0e4cf1982850389149d980 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/5f62da378d0e4cf1982850389149d980 2024-11-17T01:53:45,526 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/86b9d0e368744adb9d43dd7bb3c18919 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/B/86b9d0e368744adb9d43dd7bb3c18919 2024-11-17T01:53:45,528 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/1901502be46542f28ff8921d5e9f794a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/1901502be46542f28ff8921d5e9f794a 2024-11-17T01:53:45,529 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/f847cf9e3ca84666823c097542d4491c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/C/f847cf9e3ca84666823c097542d4491c 2024-11-17T01:53:45,531 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/recovered.edits/437.seqid to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9/recovered.edits/437.seqid 2024-11-17T01:53:45,531 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,532 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T01:53:45,532 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T01:53:45,533 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T01:53:45,535 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111700b397db2d2345eaac07e748c4d22348_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111700b397db2d2345eaac07e748c4d22348_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,536 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170af3bf3b5a364d559224f3af04b3616e_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170af3bf3b5a364d559224f3af04b3616e_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,537 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172491ecbcb90346e9adad7a8ca111f6ff_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172491ecbcb90346e9adad7a8ca111f6ff_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,538 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174f8a1e0bde98421b993a75974f5c9387_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174f8a1e0bde98421b993a75974f5c9387_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,540 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111754f27d6c6b4f41aa9bb9f88dea5e59c8_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111754f27d6c6b4f41aa9bb9f88dea5e59c8_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,541 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175838305bebb34024b7c8c1cd84adbf67_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175838305bebb34024b7c8c1cd84adbf67_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,542 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175e99d0f630a242bfaa0b3e234691f0b5_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175e99d0f630a242bfaa0b3e234691f0b5_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,543 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111767f02acbeeca4afabbbfce1ede687717_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111767f02acbeeca4afabbbfce1ede687717_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,543 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111773fc29e2a2464b219646cb651bfc8035_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111773fc29e2a2464b219646cb651bfc8035_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,544 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111779941aa5de544d638f61dfa32c6d7967_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111779941aa5de544d638f61dfa32c6d7967_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,544 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117950b16311baf405a8e041241f1522910_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117950b16311baf405a8e041241f1522910_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,545 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117961d0ef338cc40408d03f695e9f8c13f_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117961d0ef338cc40408d03f695e9f8c13f_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,545 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179765bb987b1147de9553fe0e9980ac83_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179765bb987b1147de9553fe0e9980ac83_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,546 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a0735bed237c4314ae6536a948dc8590_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a0735bed237c4314ae6536a948dc8590_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,547 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b29587cac3794444bc32d79cd727c7be_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b29587cac3794444bc32d79cd727c7be_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,547 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b48603dc290a408b831ef089c62f4c62_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b48603dc290a408b831ef089c62f4c62_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,548 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b5e475f89ce74e71866c2a69d54f28a9_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b5e475f89ce74e71866c2a69d54f28a9_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,548 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bfd7e14cd94847068039796fba6cf3b8_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bfd7e14cd94847068039796fba6cf3b8_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,549 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d23163c2f08c4fd49affce46c5a71337_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d23163c2f08c4fd49affce46c5a71337_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,549 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d3756efa224541f98c78ca1c1b419287_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d3756efa224541f98c78ca1c1b419287_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,550 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ef27531a4a7d4ee1a6fb1996a6ed3bc5_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ef27531a4a7d4ee1a6fb1996a6ed3bc5_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,550 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f3492866dd384e2db1ae0e5c129f81bd_fe092165677ecd92ff68c61a7627feb9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f3492866dd384e2db1ae0e5c129f81bd_fe092165677ecd92ff68c61a7627feb9 2024-11-17T01:53:45,551 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T01:53:45,552 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:53:45,554 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T01:53:45,555 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T01:53:45,556 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:53:45,556 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T01:53:45,556 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731808425556"}]},"ts":"9223372036854775807"} 2024-11-17T01:53:45,559 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T01:53:45,559 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fe092165677ecd92ff68c61a7627feb9, NAME => 'TestAcidGuarantees,,1731808390384.fe092165677ecd92ff68c61a7627feb9.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T01:53:45,559 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T01:53:45,559 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731808425559"}]},"ts":"9223372036854775807"} 2024-11-17T01:53:45,561 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T01:53:45,561 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:53:45,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-11-17T01:53:45,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T01:53:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T01:53:45,629 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:45,640 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249 (was 250), OpenFileDescriptor=445 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=464 (was 489), ProcessCount=11 (was 11), AvailableMemoryMB=5827 (was 4856) - AvailableMemoryMB LEAK? - 2024-11-17T01:53:45,648 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=464, ProcessCount=11, AvailableMemoryMB=5827 2024-11-17T01:53:45,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T01:53:45,649 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:53:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T01:53:45,651 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T01:53:45,651 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:45,652 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-11-17T01:53:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T01:53:45,653 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T01:53:45,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742269_1445 (size=1061) 2024-11-17T01:53:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T01:53:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T01:53:46,060 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1 2024-11-17T01:53:46,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742270_1446 (size=53) 2024-11-17T01:53:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T01:53:46,466 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:53:46,467 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5e67ace45ed3f9aacb3b2064d2a9351c, disabling compactions & flushes 2024-11-17T01:53:46,467 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:46,467 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:46,467 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. after waiting 0 ms 2024-11-17T01:53:46,467 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:46,467 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:46,467 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5e67ace45ed3f9aacb3b2064d2a9351c: Waiting for close lock at 1731808426467Disabling compacts and flushes for region at 1731808426467Disabling writes for close at 1731808426467Writing region close event to WAL at 1731808426467Closed at 1731808426467 2024-11-17T01:53:46,470 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T01:53:46,470 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731808426470"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731808426470"}]},"ts":"1731808426470"} 2024-11-17T01:53:46,474 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T01:53:46,476 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T01:53:46,476 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808426476"}]},"ts":"1731808426476"} 2024-11-17T01:53:46,478 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T01:53:46,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e67ace45ed3f9aacb3b2064d2a9351c, ASSIGN}] 2024-11-17T01:53:46,479 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e67ace45ed3f9aacb3b2064d2a9351c, ASSIGN 2024-11-17T01:53:46,481 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e67ace45ed3f9aacb3b2064d2a9351c, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,41157,1731808292173; forceNewPlan=false, retain=false 2024-11-17T01:53:46,631 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=5e67ace45ed3f9aacb3b2064d2a9351c, regionState=OPENING, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:46,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e67ace45ed3f9aacb3b2064d2a9351c, ASSIGN because future has completed 2024-11-17T01:53:46,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173}] 2024-11-17T01:53:46,791 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:46,791 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:53:46,792 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:46,792 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:53:46,792 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:46,792 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:46,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T01:53:46,794 INFO [StoreOpener-5e67ace45ed3f9aacb3b2064d2a9351c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:46,796 INFO [StoreOpener-5e67ace45ed3f9aacb3b2064d2a9351c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:53:46,797 INFO [StoreOpener-5e67ace45ed3f9aacb3b2064d2a9351c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e67ace45ed3f9aacb3b2064d2a9351c columnFamilyName A 2024-11-17T01:53:46,797 DEBUG [StoreOpener-5e67ace45ed3f9aacb3b2064d2a9351c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:46,798 INFO [StoreOpener-5e67ace45ed3f9aacb3b2064d2a9351c-1 {}] regionserver.HStore(327): Store=5e67ace45ed3f9aacb3b2064d2a9351c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:53:46,798 INFO [StoreOpener-5e67ace45ed3f9aacb3b2064d2a9351c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:46,799 INFO [StoreOpener-5e67ace45ed3f9aacb3b2064d2a9351c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:53:46,800 INFO [StoreOpener-5e67ace45ed3f9aacb3b2064d2a9351c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e67ace45ed3f9aacb3b2064d2a9351c columnFamilyName B 2024-11-17T01:53:46,800 DEBUG [StoreOpener-5e67ace45ed3f9aacb3b2064d2a9351c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:46,800 INFO [StoreOpener-5e67ace45ed3f9aacb3b2064d2a9351c-1 {}] regionserver.HStore(327): Store=5e67ace45ed3f9aacb3b2064d2a9351c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:53:46,801 INFO [StoreOpener-5e67ace45ed3f9aacb3b2064d2a9351c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:46,802 INFO [StoreOpener-5e67ace45ed3f9aacb3b2064d2a9351c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:53:46,802 INFO [StoreOpener-5e67ace45ed3f9aacb3b2064d2a9351c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e67ace45ed3f9aacb3b2064d2a9351c columnFamilyName C 2024-11-17T01:53:46,802 DEBUG [StoreOpener-5e67ace45ed3f9aacb3b2064d2a9351c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:53:46,803 INFO [StoreOpener-5e67ace45ed3f9aacb3b2064d2a9351c-1 {}] regionserver.HStore(327): Store=5e67ace45ed3f9aacb3b2064d2a9351c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:53:46,803 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:46,803 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:46,804 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:46,804 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:46,805 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:46,805 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:46,806 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T01:53:46,807 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:46,809 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:53:46,809 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened 5e67ace45ed3f9aacb3b2064d2a9351c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67713240, jitterRate=0.009005904197692871}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T01:53:46,809 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:46,810 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for 5e67ace45ed3f9aacb3b2064d2a9351c: Running coprocessor pre-open hook at 1731808426792Writing region info on filesystem at 1731808426793 (+1 ms)Initializing all the Stores at 1731808426794 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808426794Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808426794Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808426794Cleaning up temporary data from old regions at 1731808426805 (+11 ms)Running coprocessor post-open hooks at 1731808426809 (+4 ms)Region opened successfully at 1731808426810 (+1 ms) 2024-11-17T01:53:46,811 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., pid=122, masterSystemTime=1731808426787 2024-11-17T01:53:46,813 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:46,813 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:46,813 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=5e67ace45ed3f9aacb3b2064d2a9351c, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:46,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 because future has completed 2024-11-17T01:53:46,818 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-17T01:53:46,818 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 in 182 msec 2024-11-17T01:53:46,819 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-17T01:53:46,819 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e67ace45ed3f9aacb3b2064d2a9351c, ASSIGN in 340 msec 2024-11-17T01:53:46,820 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T01:53:46,820 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808426820"}]},"ts":"1731808426820"} 2024-11-17T01:53:46,821 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T01:53:46,822 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T01:53:46,823 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1730 sec 2024-11-17T01:53:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T01:53:47,805 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:47,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8abc8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:47,806 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:47,807 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58cf5906, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:47,808 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:47,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b4a492, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,809 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:47,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:47,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:47,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:47,811 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T01:53:47,812 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is c6c0d7a0a7c0,45023,1731808291372 2024-11-17T01:53:47,812 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@38259f36 2024-11-17T01:53:47,812 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T01:53:47,813 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T01:53:47,814 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bd122f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,814 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:47,815 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:47,815 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:47,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:47,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55cfb594, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:47,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:47,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:47,817 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@421fa67b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,818 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:47,818 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:47,819 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:47,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:47,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:47,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d9718ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:47,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:47,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:47,820 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:47,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@261e1404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,821 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,821 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:47,821 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:47,822 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30197932, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:47,823 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:47,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dcc23a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,824 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:47,824 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:47,824 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52b57b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:47,826 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e8f8c03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,827 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:47,827 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:47,827 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f125826, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:47,828 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30c73ab2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:47,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:47,830 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:47,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:47,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:47,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60461f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:47,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:47,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:47,831 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:47,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc8891a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,832 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:47,832 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:47,833 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:47,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:47,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:47,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76b7e9ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:47,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:47,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:47,834 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:47,834 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1044369, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,835 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:47,835 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:47,835 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:47,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:47,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:47,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ae1d311, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:47,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:47,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:47,837 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45fe269e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,838 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:47,838 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:47,839 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28a3474a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:47,840 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:47,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e7f7a2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:53:47,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:53:47,842 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:53:47,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:53:47,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:53:47,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771cf962, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:53:47,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:53:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:53:47,843 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:53:47,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40adfd76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:53:47,844 DEBUG [Thread-2014 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:47,844 DEBUG [Thread-2018 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:47,844 DEBUG [Thread-2003 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:47,844 DEBUG [Thread-2007 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:47,844 DEBUG [Thread-2001 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:47,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:47,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:47,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:47,845 DEBUG [Thread-2012 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:47,845 DEBUG [Thread-2014 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:47,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:47,845 DEBUG [Thread-2003 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:47,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:47,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:47,846 DEBUG [Thread-2001 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:47,846 DEBUG [Thread-2001 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:47,846 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:47,846 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:47,847 DEBUG [Thread-2005 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:47,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:47,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:47,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:47,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:47,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:47,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:53:47,848 DEBUG [Thread-2020 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:47,849 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:53:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:47,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:47,850 DEBUG [Thread-2009 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:47,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:53:47,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:53:47,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:53:47,852 DEBUG [Thread-2016 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:53:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:47,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:53:47,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:47,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:53:47,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:53:47,854 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:47,854 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:53:47,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:53:47,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:53:47,858 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:53:47,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:53:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-17T01:53:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T01:53:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:47,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e67ace45ed3f9aacb3b2064d2a9351c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T01:53:47,861 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:53:47,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2] 2024-11-17T01:53:47,862 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:53:47,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:53:47,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=A 2024-11-17T01:53:47,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:47,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=B 2024-11-17T01:53:47,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:47,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=C 2024-11-17T01:53:47,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:47,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/d8290f7308a744009177a55b4c1f2390 is 50, key is test_row_0/A:col10/1731808427861/Put/seqid=0 2024-11-17T01:53:47,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742271_1447 (size=12001) 2024-11-17T01:53:47,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808487905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:47,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:47,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:47,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:47,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808487908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:47,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:47,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808487908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:47,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808487909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:47,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:47,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:47,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808487909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:47,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:47,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:47,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:47,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T01:53:48,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T01:53:48,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:48,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:48,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:48,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808488014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:48,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:48,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808488024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:48,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:48,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808488025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:48,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808488025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:48,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808488025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:48,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T01:53:48,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:48,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:48,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:48,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T01:53:48,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808488224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:48,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:48,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808488234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:48,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:48,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808488235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:48,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:48,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:48,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808488235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:48,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:48,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808488235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:48,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/d8290f7308a744009177a55b4c1f2390 2024-11-17T01:53:48,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/df4d9522e1774797abfb62ec94101eea is 50, key is test_row_0/B:col10/1731808427861/Put/seqid=0 2024-11-17T01:53:48,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T01:53:48,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:48,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:48,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:48,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742272_1448 (size=12001) 2024-11-17T01:53:48,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T01:53:48,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:48,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:48,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:48,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T01:53:48,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808488534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:48,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:48,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808488544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808488544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808488544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808488544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:48,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:48,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T01:53:48,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:48,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:48,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:48,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/df4d9522e1774797abfb62ec94101eea 2024-11-17T01:53:48,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/fb7359e4441544568e68fbb8dc934931 is 50, key is test_row_0/C:col10/1731808427861/Put/seqid=0 2024-11-17T01:53:48,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742273_1449 (size=12001) 2024-11-17T01:53:48,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T01:53:48,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:48,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:48,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:48,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T01:53:48,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:48,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:48,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:48,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T01:53:49,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:49,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808489044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:49,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:49,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808489064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:49,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808489064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:49,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808489064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:49,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808489064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:49,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T01:53:49,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:49,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:49,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:49,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:49,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:49,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:49,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/fb7359e4441544568e68fbb8dc934931 2024-11-17T01:53:49,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/d8290f7308a744009177a55b4c1f2390 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/d8290f7308a744009177a55b4c1f2390 2024-11-17T01:53:49,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/d8290f7308a744009177a55b4c1f2390, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T01:53:49,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/df4d9522e1774797abfb62ec94101eea as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/df4d9522e1774797abfb62ec94101eea 2024-11-17T01:53:49,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/df4d9522e1774797abfb62ec94101eea, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T01:53:49,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/fb7359e4441544568e68fbb8dc934931 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/fb7359e4441544568e68fbb8dc934931 2024-11-17T01:53:49,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/fb7359e4441544568e68fbb8dc934931, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T01:53:49,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5e67ace45ed3f9aacb3b2064d2a9351c in 1316ms, sequenceid=13, compaction requested=false 2024-11-17T01:53:49,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:49,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T01:53:49,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:49,239 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 5e67ace45ed3f9aacb3b2064d2a9351c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T01:53:49,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=A 2024-11-17T01:53:49,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:49,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=B 2024-11-17T01:53:49,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:49,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=C 2024-11-17T01:53:49,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:49,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/0cb882ec42a748cc95fd137ec6800796 is 50, key is test_row_0/A:col10/1731808427908/Put/seqid=0 2024-11-17T01:53:49,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742274_1450 (size=12001) 2024-11-17T01:53:49,652 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/0cb882ec42a748cc95fd137ec6800796 2024-11-17T01:53:49,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/4fd1682cc4f04529ad6db54770213874 is 50, key is test_row_0/B:col10/1731808427908/Put/seqid=0 2024-11-17T01:53:49,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742275_1451 (size=12001) 2024-11-17T01:53:49,692 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/4fd1682cc4f04529ad6db54770213874 2024-11-17T01:53:49,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/be1f431852c344e7a0b94d79ea12cdac is 50, key is test_row_0/C:col10/1731808427908/Put/seqid=0 2024-11-17T01:53:49,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742276_1452 (size=12001) 2024-11-17T01:53:49,708 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/be1f431852c344e7a0b94d79ea12cdac 2024-11-17T01:53:49,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/0cb882ec42a748cc95fd137ec6800796 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/0cb882ec42a748cc95fd137ec6800796 2024-11-17T01:53:49,721 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/0cb882ec42a748cc95fd137ec6800796, entries=150, sequenceid=33, filesize=11.7 K 2024-11-17T01:53:49,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/4fd1682cc4f04529ad6db54770213874 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/4fd1682cc4f04529ad6db54770213874 2024-11-17T01:53:49,726 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/4fd1682cc4f04529ad6db54770213874, entries=150, sequenceid=33, filesize=11.7 K 2024-11-17T01:53:49,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/be1f431852c344e7a0b94d79ea12cdac as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/be1f431852c344e7a0b94d79ea12cdac 2024-11-17T01:53:49,736 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/be1f431852c344e7a0b94d79ea12cdac, entries=150, sequenceid=33, filesize=11.7 K 2024-11-17T01:53:49,737 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 5e67ace45ed3f9aacb3b2064d2a9351c in 498ms, sequenceid=33, compaction requested=false 2024-11-17T01:53:49,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:49,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:49,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-17T01:53:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-17T01:53:49,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-17T01:53:49,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8760 sec 2024-11-17T01:53:49,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.8840 sec 2024-11-17T01:53:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T01:53:50,005 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:50,005 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:53:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-17T01:53:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T01:53:50,007 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:53:50,007 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:53:50,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:53:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:50,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e67ace45ed3f9aacb3b2064d2a9351c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T01:53:50,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=A 2024-11-17T01:53:50,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:50,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=B 2024-11-17T01:53:50,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:50,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=C 2024-11-17T01:53:50,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:50,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/cd780a01f0994f5782bc7b2f229128bc is 50, key is test_row_1/A:col10/1731808430060/Put/seqid=0 2024-11-17T01:53:50,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742277_1453 (size=11997) 2024-11-17T01:53:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T01:53:50,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731808430087, max timestamp=1731808430112Num uniques 859; 2024-11-17T01:53:50,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731808430087, max timestamp=1731808430112Num uniques 859; 2024-11-17T01:53:50,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T01:53:50,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T01:53:50,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731808430087, max timestamp=1731808430112Num uniques 859; 2024-11-17T01:53:50,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T01:53:50,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808490129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:50,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808490134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:50,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:50,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:50,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:50,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808490135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:50,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:50,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808490139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:50,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808490140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:50,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:50,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T01:53:50,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:50,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:50,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:50,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:50,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:50,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:50,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808490244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:50,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:50,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808490244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:50,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:50,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:50,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:50,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808490254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:50,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:50,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808490254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:50,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:50,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:50,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:50,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808490254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:50,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T01:53:50,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:50,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:50,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:50,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:50,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:50,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:50,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T01:53:50,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:50,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808490454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:50,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:50,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808490454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:50,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:50,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:50,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T01:53:50,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:50,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:50,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:50,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:50,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:50,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:50,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808490464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:50,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808490464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:50,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808490464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:50,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/cd780a01f0994f5782bc7b2f229128bc 2024-11-17T01:53:50,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/5966f7616dab418dba2fb7e0077832e6 is 50, key is test_row_1/B:col10/1731808430060/Put/seqid=0 2024-11-17T01:53:50,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742278_1454 (size=9657) 2024-11-17T01:53:50,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T01:53:50,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:50,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:50,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:50,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:50,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:50,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:50,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T01:53:50,725 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T01:53:50,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T01:53:50,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:50,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:50,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:50,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:50,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:50,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808490764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:50,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808490775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:50,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808490775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808490775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:50,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808490784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:50,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/5966f7616dab418dba2fb7e0077832e6 2024-11-17T01:53:50,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/1defd09408be4ee5b9e4e5a5ac8d065d is 50, key is test_row_1/C:col10/1731808430060/Put/seqid=0 2024-11-17T01:53:50,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742279_1455 (size=9657) 2024-11-17T01:53:50,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T01:53:50,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:50,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:50,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:50,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:50,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:51,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T01:53:51,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:51,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:51,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:51,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:51,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T01:53:51,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T01:53:51,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:51,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:51,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:51,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:51,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:51,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808491283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:51,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:51,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:51,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:51,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808491294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:51,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808491294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:51,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808491294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:51,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/1defd09408be4ee5b9e4e5a5ac8d065d 2024-11-17T01:53:51,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/cd780a01f0994f5782bc7b2f229128bc as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/cd780a01f0994f5782bc7b2f229128bc 2024-11-17T01:53:51,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/cd780a01f0994f5782bc7b2f229128bc, entries=150, sequenceid=43, filesize=11.7 K 2024-11-17T01:53:51,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/5966f7616dab418dba2fb7e0077832e6 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/5966f7616dab418dba2fb7e0077832e6 2024-11-17T01:53:51,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808491304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:51,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:51,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:51,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:51,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/5966f7616dab418dba2fb7e0077832e6, entries=100, sequenceid=43, filesize=9.4 K 2024-11-17T01:53:51,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/1defd09408be4ee5b9e4e5a5ac8d065d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/1defd09408be4ee5b9e4e5a5ac8d065d 2024-11-17T01:53:51,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/1defd09408be4ee5b9e4e5a5ac8d065d, entries=100, sequenceid=43, filesize=9.4 K 2024-11-17T01:53:51,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 5e67ace45ed3f9aacb3b2064d2a9351c in 1242ms, sequenceid=43, compaction requested=true 2024-11-17T01:53:51,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:51,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:53:51,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:51,319 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:51,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:53:51,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:51,319 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:51,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:53:51,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:51,319 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:51,319 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:51,319 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/A is initiating minor compaction (all files) 2024-11-17T01:53:51,319 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/B is initiating minor compaction (all files) 2024-11-17T01:53:51,319 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/B in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:51,319 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/A in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:51,320 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/d8290f7308a744009177a55b4c1f2390, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/0cb882ec42a748cc95fd137ec6800796, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/cd780a01f0994f5782bc7b2f229128bc] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=35.2 K 2024-11-17T01:53:51,320 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/df4d9522e1774797abfb62ec94101eea, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/4fd1682cc4f04529ad6db54770213874, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/5966f7616dab418dba2fb7e0077832e6] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=32.9 K 2024-11-17T01:53:51,320 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8290f7308a744009177a55b4c1f2390, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731808427859 2024-11-17T01:53:51,320 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting df4d9522e1774797abfb62ec94101eea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731808427859 2024-11-17T01:53:51,320 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fd1682cc4f04529ad6db54770213874, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731808427892 2024-11-17T01:53:51,320 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cb882ec42a748cc95fd137ec6800796, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731808427892 2024-11-17T01:53:51,320 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 5966f7616dab418dba2fb7e0077832e6, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731808430060 2024-11-17T01:53:51,320 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd780a01f0994f5782bc7b2f229128bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731808430060 2024-11-17T01:53:51,325 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#B#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:51,325 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#A#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:51,325 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/4c743bbec11942a88855d8e996487503 is 50, key is test_row_0/B:col10/1731808427908/Put/seqid=0 2024-11-17T01:53:51,326 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/321b7c7a99a84c9db512eb25223fb296 is 50, key is test_row_0/A:col10/1731808427908/Put/seqid=0 2024-11-17T01:53:51,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742280_1456 (size=12104) 2024-11-17T01:53:51,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742281_1457 (size=12104) 2024-11-17T01:53:51,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T01:53:51,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:51,380 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 5e67ace45ed3f9aacb3b2064d2a9351c 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-17T01:53:51,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=A 2024-11-17T01:53:51,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:51,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=B 2024-11-17T01:53:51,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:51,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=C 2024-11-17T01:53:51,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:51,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/64edfd2a4eda4cc8b22f005161848dcc is 50, key is test_row_0/A:col10/1731808430138/Put/seqid=0 2024-11-17T01:53:51,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742282_1458 (size=12001) 2024-11-17T01:53:51,732 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/4c743bbec11942a88855d8e996487503 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/4c743bbec11942a88855d8e996487503 2024-11-17T01:53:51,733 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/321b7c7a99a84c9db512eb25223fb296 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/321b7c7a99a84c9db512eb25223fb296 2024-11-17T01:53:51,737 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/B of 5e67ace45ed3f9aacb3b2064d2a9351c into 4c743bbec11942a88855d8e996487503(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:51,737 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/A of 5e67ace45ed3f9aacb3b2064d2a9351c into 321b7c7a99a84c9db512eb25223fb296(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:51,737 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:51,737 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:51,737 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/B, priority=13, startTime=1731808431319; duration=0sec 2024-11-17T01:53:51,737 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/A, priority=13, startTime=1731808431319; duration=0sec 2024-11-17T01:53:51,737 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:51,737 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:51,737 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:A 2024-11-17T01:53:51,737 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:B 2024-11-17T01:53:51,737 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:51,738 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:51,738 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/C is initiating minor compaction (all files) 2024-11-17T01:53:51,738 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/C in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:51,738 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/fb7359e4441544568e68fbb8dc934931, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/be1f431852c344e7a0b94d79ea12cdac, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/1defd09408be4ee5b9e4e5a5ac8d065d] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=32.9 K 2024-11-17T01:53:51,738 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb7359e4441544568e68fbb8dc934931, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731808427859 2024-11-17T01:53:51,738 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting be1f431852c344e7a0b94d79ea12cdac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731808427892 2024-11-17T01:53:51,738 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1defd09408be4ee5b9e4e5a5ac8d065d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731808430060 2024-11-17T01:53:51,742 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#C#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:51,743 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/cec162b18d1d43b0b7b7089c45984207 is 50, key is test_row_0/C:col10/1731808427908/Put/seqid=0 2024-11-17T01:53:51,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742283_1459 (size=12104) 2024-11-17T01:53:51,788 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/64edfd2a4eda4cc8b22f005161848dcc 2024-11-17T01:53:51,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/188cd87687444e8491f67a3c2c78f9e4 is 50, key is test_row_0/B:col10/1731808430138/Put/seqid=0 2024-11-17T01:53:51,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742284_1460 (size=12001) 2024-11-17T01:53:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T01:53:52,158 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/cec162b18d1d43b0b7b7089c45984207 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/cec162b18d1d43b0b7b7089c45984207 2024-11-17T01:53:52,163 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/C of 5e67ace45ed3f9aacb3b2064d2a9351c into cec162b18d1d43b0b7b7089c45984207(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:52,163 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:52,163 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/C, priority=13, startTime=1731808431319; duration=0sec 2024-11-17T01:53:52,163 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:52,163 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:C 2024-11-17T01:53:52,200 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/188cd87687444e8491f67a3c2c78f9e4 2024-11-17T01:53:52,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/4eade3276295457ea696e2007f8ce32e is 50, key is test_row_0/C:col10/1731808430138/Put/seqid=0 2024-11-17T01:53:52,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742285_1461 (size=12001) 2024-11-17T01:53:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:52,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:52,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808492325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:52,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808492326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:52,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:52,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:52,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808492326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808492332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:52,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:52,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808492333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:52,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:52,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:52,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:52,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808492444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:52,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808492444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:52,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:52,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:52,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808492454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:52,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:52,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808492454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:52,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:52,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:52,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808492454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:52,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:52,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:52,618 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/4eade3276295457ea696e2007f8ce32e 2024-11-17T01:53:52,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/64edfd2a4eda4cc8b22f005161848dcc as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/64edfd2a4eda4cc8b22f005161848dcc 2024-11-17T01:53:52,626 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/64edfd2a4eda4cc8b22f005161848dcc, entries=150, sequenceid=78, filesize=11.7 K 2024-11-17T01:53:52,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/188cd87687444e8491f67a3c2c78f9e4 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/188cd87687444e8491f67a3c2c78f9e4 2024-11-17T01:53:52,631 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/188cd87687444e8491f67a3c2c78f9e4, entries=150, sequenceid=78, filesize=11.7 K 2024-11-17T01:53:52,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/4eade3276295457ea696e2007f8ce32e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/4eade3276295457ea696e2007f8ce32e 2024-11-17T01:53:52,637 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/4eade3276295457ea696e2007f8ce32e, entries=150, sequenceid=78, filesize=11.7 K 2024-11-17T01:53:52,638 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=53.67 KB/54960 for 5e67ace45ed3f9aacb3b2064d2a9351c in 1257ms, sequenceid=78, compaction requested=false 2024-11-17T01:53:52,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:52,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:52,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-17T01:53:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-17T01:53:52,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-17T01:53:52,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6320 sec 2024-11-17T01:53:52,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.6360 sec 2024-11-17T01:53:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:52,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e67ace45ed3f9aacb3b2064d2a9351c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T01:53:52,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=A 2024-11-17T01:53:52,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:52,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=B 2024-11-17T01:53:52,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:52,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=C 2024-11-17T01:53:52,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:52,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/065292d58f484f4fbf0b74690c1f3e94 is 50, key is test_row_0/A:col10/1731808432668/Put/seqid=0 2024-11-17T01:53:52,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742286_1462 (size=14341) 2024-11-17T01:53:52,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808492738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:52,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808492738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:52,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:52,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:52,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808492739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:52,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808492740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:52,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808492741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:52,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808492854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:52,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808492854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:52,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808492864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:52,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808492864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808492865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:53,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:53,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808493074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:53,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:53,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808493074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:53,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:53,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:53,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:53,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:53,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:53,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:53,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/065292d58f484f4fbf0b74690c1f3e94 2024-11-17T01:53:53,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:53,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808493084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:53,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808493084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:53,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:53,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:53,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:53,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:53,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:53,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:53,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808493084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:53,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:53,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:53,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:53,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/f337b7da9bd84578ad0320036f9d235c is 50, key is test_row_0/B:col10/1731808432668/Put/seqid=0 2024-11-17T01:53:53,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742287_1463 (size=12001) 2024-11-17T01:53:53,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:53,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808493394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:53,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:53,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808493394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:53,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808493404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:53,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:53,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808493404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:53,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808493404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:53,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/f337b7da9bd84578ad0320036f9d235c 2024-11-17T01:53:53,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/958485e84cb041669bb3606ac06f4d4e is 50, key is test_row_0/C:col10/1731808432668/Put/seqid=0 2024-11-17T01:53:53,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742288_1464 (size=12001) 2024-11-17T01:53:53,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/958485e84cb041669bb3606ac06f4d4e 2024-11-17T01:53:53,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/065292d58f484f4fbf0b74690c1f3e94 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/065292d58f484f4fbf0b74690c1f3e94 2024-11-17T01:53:53,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/065292d58f484f4fbf0b74690c1f3e94, entries=200, sequenceid=93, filesize=14.0 K 2024-11-17T01:53:53,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/f337b7da9bd84578ad0320036f9d235c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f337b7da9bd84578ad0320036f9d235c 2024-11-17T01:53:53,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f337b7da9bd84578ad0320036f9d235c, entries=150, sequenceid=93, filesize=11.7 K 2024-11-17T01:53:53,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/958485e84cb041669bb3606ac06f4d4e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/958485e84cb041669bb3606ac06f4d4e 2024-11-17T01:53:53,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/958485e84cb041669bb3606ac06f4d4e, entries=150, sequenceid=93, filesize=11.7 K 2024-11-17T01:53:53,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5e67ace45ed3f9aacb3b2064d2a9351c in 866ms, sequenceid=93, compaction requested=true 2024-11-17T01:53:53,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:53,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:53:53,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:53,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:53:53,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:53,536 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:53,536 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:53,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:53:53,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:53,536 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:53,536 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:53,537 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/B is initiating minor compaction (all files) 2024-11-17T01:53:53,537 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/A is initiating minor compaction (all files) 2024-11-17T01:53:53,537 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/A in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:53,537 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/B in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:53,537 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/4c743bbec11942a88855d8e996487503, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/188cd87687444e8491f67a3c2c78f9e4, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f337b7da9bd84578ad0320036f9d235c] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=35.3 K 2024-11-17T01:53:53,537 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/321b7c7a99a84c9db512eb25223fb296, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/64edfd2a4eda4cc8b22f005161848dcc, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/065292d58f484f4fbf0b74690c1f3e94] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=37.5 K 2024-11-17T01:53:53,537 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c743bbec11942a88855d8e996487503, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731808427908 2024-11-17T01:53:53,537 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 321b7c7a99a84c9db512eb25223fb296, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731808427908 2024-11-17T01:53:53,537 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 188cd87687444e8491f67a3c2c78f9e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731808430132 2024-11-17T01:53:53,538 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64edfd2a4eda4cc8b22f005161848dcc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731808430132 2024-11-17T01:53:53,538 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 065292d58f484f4fbf0b74690c1f3e94, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731808432325 2024-11-17T01:53:53,538 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting f337b7da9bd84578ad0320036f9d235c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731808432325 2024-11-17T01:53:53,548 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#B#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:53,549 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#A#compaction#389 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:53,549 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/aa390df369d34b9a99685e93d8bdd567 is 50, key is test_row_0/B:col10/1731808432668/Put/seqid=0 2024-11-17T01:53:53,549 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/7ac7f5be4144437c9912149c070d14ee is 50, key is test_row_0/A:col10/1731808432668/Put/seqid=0 2024-11-17T01:53:53,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742289_1465 (size=12207) 2024-11-17T01:53:53,569 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/7ac7f5be4144437c9912149c070d14ee as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/7ac7f5be4144437c9912149c070d14ee 2024-11-17T01:53:53,578 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/A of 5e67ace45ed3f9aacb3b2064d2a9351c into 7ac7f5be4144437c9912149c070d14ee(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:53,578 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:53,578 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/A, priority=13, startTime=1731808433536; duration=0sec 2024-11-17T01:53:53,578 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:53,578 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:A 2024-11-17T01:53:53,578 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:53,580 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:53,580 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/C is initiating minor compaction (all files) 2024-11-17T01:53:53,580 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/C in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:53,580 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/cec162b18d1d43b0b7b7089c45984207, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/4eade3276295457ea696e2007f8ce32e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/958485e84cb041669bb3606ac06f4d4e] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=35.3 K 2024-11-17T01:53:53,580 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting cec162b18d1d43b0b7b7089c45984207, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731808427908 2024-11-17T01:53:53,581 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4eade3276295457ea696e2007f8ce32e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731808430132 2024-11-17T01:53:53,581 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 958485e84cb041669bb3606ac06f4d4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731808432325 2024-11-17T01:53:53,588 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#C#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:53,589 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/aa95cf3061024a529f122d0149b90e7f is 50, key is test_row_0/C:col10/1731808432668/Put/seqid=0 2024-11-17T01:53:53,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742290_1466 (size=12207) 2024-11-17T01:53:53,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742291_1467 (size=12207) 2024-11-17T01:53:53,630 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/aa95cf3061024a529f122d0149b90e7f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/aa95cf3061024a529f122d0149b90e7f 2024-11-17T01:53:53,636 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/aa390df369d34b9a99685e93d8bdd567 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/aa390df369d34b9a99685e93d8bdd567 2024-11-17T01:53:53,643 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/C of 5e67ace45ed3f9aacb3b2064d2a9351c into aa95cf3061024a529f122d0149b90e7f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:53,643 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:53,643 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/C, priority=13, startTime=1731808433536; duration=0sec 2024-11-17T01:53:53,643 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/B of 5e67ace45ed3f9aacb3b2064d2a9351c into aa390df369d34b9a99685e93d8bdd567(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:53,643 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:53,643 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:53,643 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/B, priority=13, startTime=1731808433536; duration=0sec 2024-11-17T01:53:53,643 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:C 2024-11-17T01:53:53,643 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:53,643 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:B 2024-11-17T01:53:53,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808432698, max timestamp=1731808433915Num uniques 859; 2024-11-17T01:53:53,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808432698, max timestamp=1731808433915Num uniques 859; 2024-11-17T01:53:53,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:53:53,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:53:53,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808432698, max timestamp=1731808433915Num uniques 859; 2024-11-17T01:53:53,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:53:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:53,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e67ace45ed3f9aacb3b2064d2a9351c 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-17T01:53:53,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=A 2024-11-17T01:53:53,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:53,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=B 2024-11-17T01:53:53,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:53,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=C 2024-11-17T01:53:53,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:53,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/c07b8a4a48c6480098c6ccee4fd26283 is 50, key is test_row_0/A:col10/1731808432739/Put/seqid=0 2024-11-17T01:53:53,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742292_1468 (size=19021) 2024-11-17T01:53:53,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731808433934, max timestamp=1731808433983Num uniques 859; 2024-11-17T01:53:53,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731808433934, max timestamp=1731808433983Num uniques 859; 2024-11-17T01:53:53,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T01:53:53,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T01:53:53,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808433934, max timestamp=1731808433983Num uniques 859; 2024-11-17T01:53:53,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:53:54,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808494035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808494035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808494037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808494037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808494038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T01:53:54,174 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:54,175 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:53:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-17T01:53:54,176 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:53:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T01:53:54,177 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:53:54,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:53:54,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808494164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808494165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808494165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808494165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808494165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T01:53:54,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T01:53:54,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:54,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:54,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:54,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:54,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:54,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/c07b8a4a48c6480098c6ccee4fd26283 2024-11-17T01:53:54,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/14e50ca752914036b3847d44a6b4ee5a is 50, key is test_row_0/B:col10/1731808432739/Put/seqid=0 2024-11-17T01:53:54,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742293_1469 (size=12001) 2024-11-17T01:53:54,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/14e50ca752914036b3847d44a6b4ee5a 2024-11-17T01:53:54,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808494385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808494385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808494385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/ed6bdf158add41129b0a68a60353495d is 50, key is test_row_0/C:col10/1731808432739/Put/seqid=0 2024-11-17T01:53:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808494385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808494385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742294_1470 (size=12001) 2024-11-17T01:53:54,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/ed6bdf158add41129b0a68a60353495d 2024-11-17T01:53:54,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/c07b8a4a48c6480098c6ccee4fd26283 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/c07b8a4a48c6480098c6ccee4fd26283 2024-11-17T01:53:54,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/c07b8a4a48c6480098c6ccee4fd26283, entries=300, sequenceid=121, filesize=18.6 K 2024-11-17T01:53:54,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/14e50ca752914036b3847d44a6b4ee5a as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/14e50ca752914036b3847d44a6b4ee5a 2024-11-17T01:53:54,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/14e50ca752914036b3847d44a6b4ee5a, entries=150, sequenceid=121, filesize=11.7 K 2024-11-17T01:53:54,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/ed6bdf158add41129b0a68a60353495d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/ed6bdf158add41129b0a68a60353495d 2024-11-17T01:53:54,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/ed6bdf158add41129b0a68a60353495d, entries=150, sequenceid=121, filesize=11.7 K 2024-11-17T01:53:54,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=131.94 KB/135110 for 5e67ace45ed3f9aacb3b2064d2a9351c in 512ms, sequenceid=121, compaction requested=false 2024-11-17T01:53:54,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:54,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T01:53:54,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:54,485 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 5e67ace45ed3f9aacb3b2064d2a9351c 3/3 column families, dataSize=131.94 KB heapSize=370.53 KB 2024-11-17T01:53:54,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=A 2024-11-17T01:53:54,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:54,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=B 2024-11-17T01:53:54,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:54,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=C 2024-11-17T01:53:54,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:53:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T01:53:54,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/925e3d65a16a4680b0affe0d20ebd2f5 is 50, key is test_row_0/A:col10/1731808434036/Put/seqid=0 2024-11-17T01:53:54,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742295_1471 (size=12151) 2024-11-17T01:53:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:54,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:54,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808494715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808494716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808494720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808494721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808494733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T01:53:54,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808494844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808494844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808494844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808494848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808494848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:54,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:54,963 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/925e3d65a16a4680b0affe0d20ebd2f5 2024-11-17T01:53:54,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/f8077d8d34bb47a7bddddcbbbbcd5694 is 50, key is test_row_0/B:col10/1731808434036/Put/seqid=0 2024-11-17T01:53:55,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742296_1472 (size=12151) 2024-11-17T01:53:55,006 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/f8077d8d34bb47a7bddddcbbbbcd5694 2024-11-17T01:53:55,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/16826ff27aa64d21bbaee7003c8d45c2 is 50, key is test_row_0/C:col10/1731808434036/Put/seqid=0 2024-11-17T01:53:55,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742297_1473 (size=12151) 2024-11-17T01:53:55,017 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/16826ff27aa64d21bbaee7003c8d45c2 2024-11-17T01:53:55,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/925e3d65a16a4680b0affe0d20ebd2f5 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/925e3d65a16a4680b0affe0d20ebd2f5 2024-11-17T01:53:55,030 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/925e3d65a16a4680b0affe0d20ebd2f5, entries=150, sequenceid=156, filesize=11.9 K 2024-11-17T01:53:55,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/f8077d8d34bb47a7bddddcbbbbcd5694 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f8077d8d34bb47a7bddddcbbbbcd5694 2024-11-17T01:53:55,035 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f8077d8d34bb47a7bddddcbbbbcd5694, entries=150, sequenceid=156, filesize=11.9 K 2024-11-17T01:53:55,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/16826ff27aa64d21bbaee7003c8d45c2 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/16826ff27aa64d21bbaee7003c8d45c2 2024-11-17T01:53:55,039 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/16826ff27aa64d21bbaee7003c8d45c2, entries=150, sequenceid=156, filesize=11.9 K 2024-11-17T01:53:55,041 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~370.48 KB/379376, currentSize=53.67 KB/54960 for 5e67ace45ed3f9aacb3b2064d2a9351c in 556ms, sequenceid=156, compaction requested=true 2024-11-17T01:53:55,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:55,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:55,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-17T01:53:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-17T01:53:55,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-17T01:53:55,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 866 msec 2024-11-17T01:53:55,045 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 869 msec 2024-11-17T01:53:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:55,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e67ace45ed3f9aacb3b2064d2a9351c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T01:53:55,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=A 2024-11-17T01:53:55,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:55,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=B 2024-11-17T01:53:55,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:55,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=C 2024-11-17T01:53:55,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:55,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/e83342ec7cd84edaa3866d474934c8c4 is 50, key is test_row_0/A:col10/1731808434714/Put/seqid=0 2024-11-17T01:53:55,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742298_1474 (size=14541) 2024-11-17T01:53:55,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808495095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:55,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:55,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808495097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:55,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808495108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:55,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:55,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808495109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:55,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808495111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:55,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:55,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:55,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:55,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808495224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:55,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808495224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:55,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:55,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808495224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:55,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:55,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808495234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:55,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:55,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808495234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:55,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T01:53:55,314 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:55,314 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:53:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-17T01:53:55,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T01:53:55,316 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:53:55,317 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:53:55,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:53:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T01:53:55,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808495444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:55,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808495444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:55,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808495445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:55,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808495445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:55,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808495445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:55,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:55,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T01:53:55,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:55,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:55,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:55,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:55,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:55,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/e83342ec7cd84edaa3866d474934c8c4 2024-11-17T01:53:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:55,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/0da5be46049148b8a67164e04d14a1a6 is 50, key is test_row_0/B:col10/1731808434714/Put/seqid=0 2024-11-17T01:53:55,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742299_1475 (size=12151) 2024-11-17T01:53:55,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T01:53:55,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:55,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:55,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:55,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:55,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T01:53:55,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808495764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:55,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:55,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808495764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:55,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:55,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808495764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808495764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808495764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:55,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T01:53:55,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:55,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:55,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:55,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:55,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:55,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/0da5be46049148b8a67164e04d14a1a6 2024-11-17T01:53:55,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/74a2445220ac4b93966f223d2123cb7b is 50, key is test_row_0/C:col10/1731808434714/Put/seqid=0 2024-11-17T01:53:55,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742300_1476 (size=12151) 2024-11-17T01:53:55,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T01:53:55,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:55,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:55,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:55,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:55,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T01:53:56,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T01:53:56,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:56,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:56,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:56,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:56,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:56,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T01:53:56,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:56,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:56,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:56,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:56,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:56,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:56,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808496274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:56,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808496284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:56,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808496284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:56,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808496285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:56,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808496285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:56,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:56,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/74a2445220ac4b93966f223d2123cb7b 2024-11-17T01:53:56,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/e83342ec7cd84edaa3866d474934c8c4 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/e83342ec7cd84edaa3866d474934c8c4 2024-11-17T01:53:56,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/e83342ec7cd84edaa3866d474934c8c4, entries=200, sequenceid=168, filesize=14.2 K 2024-11-17T01:53:56,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/0da5be46049148b8a67164e04d14a1a6 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/0da5be46049148b8a67164e04d14a1a6 2024-11-17T01:53:56,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/0da5be46049148b8a67164e04d14a1a6, entries=150, sequenceid=168, filesize=11.9 K 2024-11-17T01:53:56,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/74a2445220ac4b93966f223d2123cb7b as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/74a2445220ac4b93966f223d2123cb7b 2024-11-17T01:53:56,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/74a2445220ac4b93966f223d2123cb7b, entries=150, sequenceid=168, filesize=11.9 K 2024-11-17T01:53:56,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5e67ace45ed3f9aacb3b2064d2a9351c in 1250ms, sequenceid=168, compaction requested=true 2024-11-17T01:53:56,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:56,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:53:56,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:56,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:53:56,310 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:53:56,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:56,310 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:53:56,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:53:56,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:56,311 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:53:56,311 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/A is initiating minor compaction (all files) 2024-11-17T01:53:56,311 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/A in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:56,311 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/7ac7f5be4144437c9912149c070d14ee, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/c07b8a4a48c6480098c6ccee4fd26283, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/925e3d65a16a4680b0affe0d20ebd2f5, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/e83342ec7cd84edaa3866d474934c8c4] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=56.6 K 2024-11-17T01:53:56,311 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:53:56,311 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ac7f5be4144437c9912149c070d14ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731808432325 2024-11-17T01:53:56,311 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/B is initiating minor compaction (all files) 2024-11-17T01:53:56,311 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/B in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:56,311 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/aa390df369d34b9a99685e93d8bdd567, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/14e50ca752914036b3847d44a6b4ee5a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f8077d8d34bb47a7bddddcbbbbcd5694, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/0da5be46049148b8a67164e04d14a1a6] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=47.4 K 2024-11-17T01:53:56,311 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting c07b8a4a48c6480098c6ccee4fd26283, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731808432739 2024-11-17T01:53:56,312 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting aa390df369d34b9a99685e93d8bdd567, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731808432325 2024-11-17T01:53:56,312 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 925e3d65a16a4680b0affe0d20ebd2f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731808434015 2024-11-17T01:53:56,312 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 14e50ca752914036b3847d44a6b4ee5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731808432739 2024-11-17T01:53:56,312 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting e83342ec7cd84edaa3866d474934c8c4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731808434714 2024-11-17T01:53:56,312 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting f8077d8d34bb47a7bddddcbbbbcd5694, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731808434015 2024-11-17T01:53:56,312 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 0da5be46049148b8a67164e04d14a1a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731808434714 2024-11-17T01:53:56,320 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#A#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:56,320 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#B#compaction#401 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:56,320 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/1aa3bada11bc47d2aeb4e1206ed77a69 is 50, key is test_row_0/A:col10/1731808434714/Put/seqid=0 2024-11-17T01:53:56,320 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/47253a3c8309487697230b44ef1a30d9 is 50, key is test_row_0/B:col10/1731808434714/Put/seqid=0 2024-11-17T01:53:56,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742302_1478 (size=12493) 2024-11-17T01:53:56,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742301_1477 (size=12493) 2024-11-17T01:53:56,333 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/1aa3bada11bc47d2aeb4e1206ed77a69 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/1aa3bada11bc47d2aeb4e1206ed77a69 2024-11-17T01:53:56,337 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/A of 5e67ace45ed3f9aacb3b2064d2a9351c into 1aa3bada11bc47d2aeb4e1206ed77a69(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:56,337 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:56,337 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/A, priority=12, startTime=1731808436310; duration=0sec 2024-11-17T01:53:56,337 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:56,337 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:A 2024-11-17T01:53:56,337 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:53:56,339 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:53:56,339 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/C is initiating minor compaction (all files) 2024-11-17T01:53:56,339 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/C in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:56,339 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/aa95cf3061024a529f122d0149b90e7f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/ed6bdf158add41129b0a68a60353495d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/16826ff27aa64d21bbaee7003c8d45c2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/74a2445220ac4b93966f223d2123cb7b] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=47.4 K 2024-11-17T01:53:56,347 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa95cf3061024a529f122d0149b90e7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731808432325 2024-11-17T01:53:56,348 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed6bdf158add41129b0a68a60353495d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731808432739 2024-11-17T01:53:56,349 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16826ff27aa64d21bbaee7003c8d45c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731808434015 2024-11-17T01:53:56,349 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74a2445220ac4b93966f223d2123cb7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731808434714 2024-11-17T01:53:56,360 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#C#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:56,360 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/d60b336b277f406e8d182d1e333e928e is 50, key is test_row_0/C:col10/1731808434714/Put/seqid=0 2024-11-17T01:53:56,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742303_1479 (size=12493) 2024-11-17T01:53:56,370 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/d60b336b277f406e8d182d1e333e928e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/d60b336b277f406e8d182d1e333e928e 2024-11-17T01:53:56,375 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/C of 5e67ace45ed3f9aacb3b2064d2a9351c into d60b336b277f406e8d182d1e333e928e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:56,375 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:56,375 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/C, priority=12, startTime=1731808436310; duration=0sec 2024-11-17T01:53:56,375 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:56,375 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:C 2024-11-17T01:53:56,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T01:53:56,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:56,390 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 5e67ace45ed3f9aacb3b2064d2a9351c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T01:53:56,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=A 2024-11-17T01:53:56,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:56,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=B 2024-11-17T01:53:56,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:56,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=C 2024-11-17T01:53:56,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:56,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/4accbf13848a41c39295c8e05aa85b48 is 50, key is test_row_0/A:col10/1731808435109/Put/seqid=0 2024-11-17T01:53:56,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742304_1480 (size=12151) 2024-11-17T01:53:56,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T01:53:56,735 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/47253a3c8309487697230b44ef1a30d9 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/47253a3c8309487697230b44ef1a30d9 2024-11-17T01:53:56,739 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/B of 5e67ace45ed3f9aacb3b2064d2a9351c into 47253a3c8309487697230b44ef1a30d9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:56,739 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:56,739 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/B, priority=12, startTime=1731808436310; duration=0sec 2024-11-17T01:53:56,739 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:56,739 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:B 2024-11-17T01:53:56,804 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/4accbf13848a41c39295c8e05aa85b48 2024-11-17T01:53:56,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/b7025af553ab4635b540b73d49d513b3 is 50, key is test_row_0/B:col10/1731808435109/Put/seqid=0 2024-11-17T01:53:56,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742305_1481 (size=12151) 2024-11-17T01:53:57,212 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/b7025af553ab4635b540b73d49d513b3 2024-11-17T01:53:57,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/980205ee49c14967b0741635219d7bf9 is 50, key is test_row_0/C:col10/1731808435109/Put/seqid=0 2024-11-17T01:53:57,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742306_1482 (size=12151) 2024-11-17T01:53:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:57,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:57,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:57,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808497310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:57,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808497311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:57,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808497314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:57,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808497321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:57,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808497321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:57,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808497424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:57,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808497434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808497434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808497434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808497434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T01:53:57,624 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/980205ee49c14967b0741635219d7bf9 2024-11-17T01:53:57,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/4accbf13848a41c39295c8e05aa85b48 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/4accbf13848a41c39295c8e05aa85b48 2024-11-17T01:53:57,630 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/4accbf13848a41c39295c8e05aa85b48, entries=150, sequenceid=190, filesize=11.9 K 2024-11-17T01:53:57,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/b7025af553ab4635b540b73d49d513b3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/b7025af553ab4635b540b73d49d513b3 2024-11-17T01:53:57,633 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/b7025af553ab4635b540b73d49d513b3, entries=150, sequenceid=190, filesize=11.9 K 2024-11-17T01:53:57,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/980205ee49c14967b0741635219d7bf9 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/980205ee49c14967b0741635219d7bf9 2024-11-17T01:53:57,636 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/980205ee49c14967b0741635219d7bf9, entries=150, sequenceid=190, filesize=11.9 K 2024-11-17T01:53:57,637 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 5e67ace45ed3f9aacb3b2064d2a9351c in 1247ms, sequenceid=190, compaction requested=false 2024-11-17T01:53:57,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:57,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:57,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-17T01:53:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-17T01:53:57,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-17T01:53:57,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3210 sec 2024-11-17T01:53:57,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.3260 sec 2024-11-17T01:53:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:57,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e67ace45ed3f9aacb3b2064d2a9351c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T01:53:57,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=A 2024-11-17T01:53:57,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:57,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=B 2024-11-17T01:53:57,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:57,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=C 2024-11-17T01:53:57,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:57,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/1d24c84cf91844939750abbca603dc6f is 50, key is test_row_0/A:col10/1731808437319/Put/seqid=0 2024-11-17T01:53:57,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742307_1483 (size=14541) 2024-11-17T01:53:57,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:57,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808497688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:57,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:57,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808497688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:57,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:57,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:57,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:57,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808497689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:57,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808497689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:57,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808497690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:57,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808497804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:57,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:57,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808497804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:57,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:57,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:57,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808497804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:57,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808497804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:57,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808497804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:58,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:58,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808498014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:58,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:58,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808498014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:58,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:58,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:58,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:58,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808498014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:58,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808498014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:58,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808498014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:58,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/1d24c84cf91844939750abbca603dc6f 2024-11-17T01:53:58,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/9ad39733030e4690943ba2ab03a51593 is 50, key is test_row_0/B:col10/1731808437319/Put/seqid=0 2024-11-17T01:53:58,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742308_1484 (size=12151) 2024-11-17T01:53:58,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/9ad39733030e4690943ba2ab03a51593 2024-11-17T01:53:58,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/d3e2c8cf94904ce58e30070151fa6e24 is 50, key is test_row_0/C:col10/1731808437319/Put/seqid=0 2024-11-17T01:53:58,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742309_1485 (size=12151) 2024-11-17T01:53:58,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808498324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:58,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:58,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808498334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:58,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:58,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808498334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:58,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:58,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:58,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808498334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:58,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:58,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:58,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808498334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:58,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:58,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:58,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/d3e2c8cf94904ce58e30070151fa6e24 2024-11-17T01:53:58,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/1d24c84cf91844939750abbca603dc6f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/1d24c84cf91844939750abbca603dc6f 2024-11-17T01:53:58,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/1d24c84cf91844939750abbca603dc6f, entries=200, sequenceid=205, filesize=14.2 K 2024-11-17T01:53:58,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/9ad39733030e4690943ba2ab03a51593 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/9ad39733030e4690943ba2ab03a51593 2024-11-17T01:53:58,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/9ad39733030e4690943ba2ab03a51593, entries=150, sequenceid=205, filesize=11.9 K 2024-11-17T01:53:58,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/d3e2c8cf94904ce58e30070151fa6e24 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/d3e2c8cf94904ce58e30070151fa6e24 2024-11-17T01:53:58,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/d3e2c8cf94904ce58e30070151fa6e24, entries=150, sequenceid=205, filesize=11.9 K 2024-11-17T01:53:58,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5e67ace45ed3f9aacb3b2064d2a9351c in 840ms, sequenceid=205, compaction requested=true 2024-11-17T01:53:58,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:58,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:53:58,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:58,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:53:58,483 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:58,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:58,483 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:58,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:53:58,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:58,483 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:58,483 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:58,483 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/A is initiating minor compaction (all files) 2024-11-17T01:53:58,483 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/B is initiating minor compaction (all files) 2024-11-17T01:53:58,483 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/A in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:58,483 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/B in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:58,484 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/47253a3c8309487697230b44ef1a30d9, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/b7025af553ab4635b540b73d49d513b3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/9ad39733030e4690943ba2ab03a51593] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=35.9 K 2024-11-17T01:53:58,484 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/1aa3bada11bc47d2aeb4e1206ed77a69, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/4accbf13848a41c39295c8e05aa85b48, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/1d24c84cf91844939750abbca603dc6f] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=38.3 K 2024-11-17T01:53:58,486 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1aa3bada11bc47d2aeb4e1206ed77a69, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731808434714 2024-11-17T01:53:58,486 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 47253a3c8309487697230b44ef1a30d9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731808434714 2024-11-17T01:53:58,486 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4accbf13848a41c39295c8e05aa85b48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731808435107 2024-11-17T01:53:58,486 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting b7025af553ab4635b540b73d49d513b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731808435107 2024-11-17T01:53:58,486 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d24c84cf91844939750abbca603dc6f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731808437312 2024-11-17T01:53:58,486 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ad39733030e4690943ba2ab03a51593, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731808437319 2024-11-17T01:53:58,491 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#B#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:58,492 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/6c0a161a4ea94501ad2092ed7a02a7d3 is 50, key is test_row_0/B:col10/1731808437319/Put/seqid=0 2024-11-17T01:53:58,492 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#A#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:58,492 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/dea26d96b03f4240a0f817ff6ef80d06 is 50, key is test_row_0/A:col10/1731808437319/Put/seqid=0 2024-11-17T01:53:58,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742310_1486 (size=12595) 2024-11-17T01:53:58,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742311_1487 (size=12595) 2024-11-17T01:53:58,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:53:58,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e67ace45ed3f9aacb3b2064d2a9351c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T01:53:58,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=A 2024-11-17T01:53:58,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:58,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=B 2024-11-17T01:53:58,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:58,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=C 2024-11-17T01:53:58,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:53:58,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/963875ab5ab84cf887817dadbc812106 is 50, key is test_row_0/A:col10/1731808437687/Put/seqid=0 2024-11-17T01:53:58,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742312_1488 (size=14541) 2024-11-17T01:53:58,904 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/6c0a161a4ea94501ad2092ed7a02a7d3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/6c0a161a4ea94501ad2092ed7a02a7d3 2024-11-17T01:53:58,907 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/B of 5e67ace45ed3f9aacb3b2064d2a9351c into 6c0a161a4ea94501ad2092ed7a02a7d3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:58,907 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:58,908 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/B, priority=13, startTime=1731808438483; duration=0sec 2024-11-17T01:53:58,908 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:53:58,908 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:B 2024-11-17T01:53:58,908 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:53:58,909 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/dea26d96b03f4240a0f817ff6ef80d06 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/dea26d96b03f4240a0f817ff6ef80d06 2024-11-17T01:53:58,910 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:53:58,910 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/C is initiating minor compaction (all files) 2024-11-17T01:53:58,910 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/C in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:58,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:58,911 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/d60b336b277f406e8d182d1e333e928e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/980205ee49c14967b0741635219d7bf9, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/d3e2c8cf94904ce58e30070151fa6e24] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=35.9 K 2024-11-17T01:53:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808498891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:58,911 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting d60b336b277f406e8d182d1e333e928e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731808434714 2024-11-17T01:53:58,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:58,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808498891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:58,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:58,911 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 980205ee49c14967b0741635219d7bf9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731808435107 2024-11-17T01:53:58,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808498892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:58,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:58,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808498892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:58,912 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting d3e2c8cf94904ce58e30070151fa6e24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731808437319 2024-11-17T01:53:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:58,913 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/A of 5e67ace45ed3f9aacb3b2064d2a9351c into dea26d96b03f4240a0f817ff6ef80d06(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:58,913 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:58,913 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/A, priority=13, startTime=1731808438483; duration=0sec 2024-11-17T01:53:58,913 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:58,913 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:A 2024-11-17T01:53:58,917 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#C#compaction#412 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:53:58,917 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/cabe72e632f142a682f3d60eb25e0556 is 50, key is test_row_0/C:col10/1731808437319/Put/seqid=0 2024-11-17T01:53:58,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742313_1489 (size=12595) 2024-11-17T01:53:58,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808498911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:59,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:59,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808499014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:59,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808499014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:59,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808499014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:59,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808499014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:59,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:59,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808499034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:59,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808499224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:59,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808499224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:59,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808499225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:59,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808499225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:59,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:59,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/963875ab5ab84cf887817dadbc812106 2024-11-17T01:53:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808499244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:59,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/0ec8ce82a8cb460eb254d39901fa6b62 is 50, key is test_row_0/B:col10/1731808437687/Put/seqid=0 2024-11-17T01:53:59,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742314_1490 (size=12151) 2024-11-17T01:53:59,324 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/cabe72e632f142a682f3d60eb25e0556 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/cabe72e632f142a682f3d60eb25e0556 2024-11-17T01:53:59,327 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/C of 5e67ace45ed3f9aacb3b2064d2a9351c into cabe72e632f142a682f3d60eb25e0556(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:53:59,327 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:53:59,327 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/C, priority=13, startTime=1731808438483; duration=0sec 2024-11-17T01:53:59,328 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:53:59,328 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:C 2024-11-17T01:53:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T01:53:59,474 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:53:59,474 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:53:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-17T01:53:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T01:53:59,476 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:53:59,476 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:53:59,476 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:53:59,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808499534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:59,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808499534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:59,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:59,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808499535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:59,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:59,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808499544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:59,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:59,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:53:59,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808499554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:53:59,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:53:59,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:53:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T01:53:59,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T01:53:59,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:59,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:59,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:59,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:59,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:59,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/0ec8ce82a8cb460eb254d39901fa6b62 2024-11-17T01:53:59,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/fdfd09a1f74a489fa9842cfe9a851c58 is 50, key is test_row_0/C:col10/1731808437687/Put/seqid=0 2024-11-17T01:53:59,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742315_1491 (size=12151) 2024-11-17T01:53:59,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T01:53:59,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:59,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:59,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:59,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:59,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:59,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:59,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T01:53:59,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T01:53:59,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:59,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:53:59,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:53:59,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:59,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:53:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:00,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808500054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:00,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:00,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:00,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808500054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:00,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:00,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:00,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:00,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808500054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:00,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:00,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:00,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:00,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:00,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808500054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:00,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:00,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:00,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:00,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808500064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:00,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:00,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:00,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:00,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/fdfd09a1f74a489fa9842cfe9a851c58 2024-11-17T01:54:00,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/963875ab5ab84cf887817dadbc812106 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/963875ab5ab84cf887817dadbc812106 2024-11-17T01:54:00,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/963875ab5ab84cf887817dadbc812106, entries=200, sequenceid=224, filesize=14.2 K 2024-11-17T01:54:00,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/0ec8ce82a8cb460eb254d39901fa6b62 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/0ec8ce82a8cb460eb254d39901fa6b62 2024-11-17T01:54:00,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/0ec8ce82a8cb460eb254d39901fa6b62, entries=150, sequenceid=224, filesize=11.9 K 2024-11-17T01:54:00,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/fdfd09a1f74a489fa9842cfe9a851c58 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/fdfd09a1f74a489fa9842cfe9a851c58 2024-11-17T01:54:00,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/fdfd09a1f74a489fa9842cfe9a851c58, entries=150, sequenceid=224, filesize=11.9 K 2024-11-17T01:54:00,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5e67ace45ed3f9aacb3b2064d2a9351c in 1243ms, sequenceid=224, compaction requested=false 2024-11-17T01:54:00,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:54:00,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T01:54:00,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:00,086 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 5e67ace45ed3f9aacb3b2064d2a9351c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T01:54:00,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=A 2024-11-17T01:54:00,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:00,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=B 2024-11-17T01:54:00,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:00,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=C 2024-11-17T01:54:00,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:00,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/424be89a8ecf4479a70e60c10968bfca is 50, key is test_row_0/A:col10/1731808438907/Put/seqid=0 2024-11-17T01:54:00,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742316_1492 (size=12151) 2024-11-17T01:54:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T01:54:00,495 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/424be89a8ecf4479a70e60c10968bfca 2024-11-17T01:54:00,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/f888be73f069469aabbfb072e2de6414 is 50, key is test_row_0/B:col10/1731808438907/Put/seqid=0 2024-11-17T01:54:00,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742317_1493 (size=12151) 2024-11-17T01:54:00,503 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T01:54:00,504 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/f888be73f069469aabbfb072e2de6414 2024-11-17T01:54:00,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/ab4fed1b0ad74dae9954985d6d75d192 is 50, key is test_row_0/C:col10/1731808438907/Put/seqid=0 2024-11-17T01:54:00,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742318_1494 (size=12151) 2024-11-17T01:54:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T01:54:00,979 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/ab4fed1b0ad74dae9954985d6d75d192 2024-11-17T01:54:00,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/424be89a8ecf4479a70e60c10968bfca as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/424be89a8ecf4479a70e60c10968bfca 2024-11-17T01:54:00,989 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/424be89a8ecf4479a70e60c10968bfca, entries=150, sequenceid=240, filesize=11.9 K 2024-11-17T01:54:00,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/f888be73f069469aabbfb072e2de6414 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f888be73f069469aabbfb072e2de6414 2024-11-17T01:54:00,993 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f888be73f069469aabbfb072e2de6414, entries=150, sequenceid=240, filesize=11.9 K 2024-11-17T01:54:00,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/ab4fed1b0ad74dae9954985d6d75d192 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/ab4fed1b0ad74dae9954985d6d75d192 2024-11-17T01:54:00,996 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/ab4fed1b0ad74dae9954985d6d75d192, entries=150, sequenceid=240, filesize=11.9 K 2024-11-17T01:54:00,997 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 5e67ace45ed3f9aacb3b2064d2a9351c in 911ms, sequenceid=240, compaction requested=true 2024-11-17T01:54:00,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:54:00,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:00,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-17T01:54:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-17T01:54:01,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-17T01:54:01,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5220 sec 2024-11-17T01:54:01,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.5270 sec 2024-11-17T01:54:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:54:01,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e67ace45ed3f9aacb3b2064d2a9351c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T01:54:01,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=A 2024-11-17T01:54:01,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:01,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=B 2024-11-17T01:54:01,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:01,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=C 2024-11-17T01:54:01,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:01,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/fb098d231c8a426fab345ddfe8cc9016 is 50, key is test_row_0/A:col10/1731808441078/Put/seqid=0 2024-11-17T01:54:01,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742319_1495 (size=19321) 2024-11-17T01:54:01,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808501116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:01,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808501117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:01,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808501117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:01,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731808441095, max timestamp=1731808441130Num uniques 859; 2024-11-17T01:54:01,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731808441095, max timestamp=1731808441130Num uniques 859; 2024-11-17T01:54:01,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T01:54:01,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T01:54:01,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731808441095, max timestamp=1731808441130Num uniques 859; 2024-11-17T01:54:01,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T01:54:01,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:01,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808501199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:01,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:01,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808501202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:01,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:01,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808501244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:01,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:01,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808501244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:01,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:01,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:01,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808501244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:01,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:01,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:01,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808501314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:01,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808501324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:01,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808501464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:01,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808501464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:01,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:01,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:01,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808501464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:01,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:01,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/fb098d231c8a426fab345ddfe8cc9016 2024-11-17T01:54:01,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/df41d6cb9d454bc3a5b46db8752ce4c8 is 50, key is test_row_0/B:col10/1731808441078/Put/seqid=0 2024-11-17T01:54:01,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742320_1496 (size=12151) 2024-11-17T01:54:01,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808501534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:01,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808501544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:01,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T01:54:01,624 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:54:01,624 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:54:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-17T01:54:01,626 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:54:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T01:54:01,627 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:54:01,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:54:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T01:54:01,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T01:54:01,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:01,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:54:01,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:01,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:01,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:01,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808501774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:01,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808501774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808501774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:01,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808501854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:01,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:01,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808501864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:01,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:01,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:01,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/df41d6cb9d454bc3a5b46db8752ce4c8 2024-11-17T01:54:01,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/0c7cbbcb37af42adbea8f634f861f073 is 50, key is test_row_0/C:col10/1731808441078/Put/seqid=0 2024-11-17T01:54:01,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742321_1497 (size=12151) 2024-11-17T01:54:01,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T01:54:01,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:01,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:54:01,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:01,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:01,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T01:54:02,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T01:54:02,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:02,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:54:02,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:02,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:02,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:02,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T01:54:02,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:02,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:54:02,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:02,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:02,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:02,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:02,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T01:54:02,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808502294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:02,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808502294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:02,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:02,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808502294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:02,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/0c7cbbcb37af42adbea8f634f861f073 2024-11-17T01:54:02,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/fb098d231c8a426fab345ddfe8cc9016 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/fb098d231c8a426fab345ddfe8cc9016 2024-11-17T01:54:02,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/fb098d231c8a426fab345ddfe8cc9016, entries=300, sequenceid=251, filesize=18.9 K 2024-11-17T01:54:02,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/df41d6cb9d454bc3a5b46db8752ce4c8 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/df41d6cb9d454bc3a5b46db8752ce4c8 2024-11-17T01:54:02,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/df41d6cb9d454bc3a5b46db8752ce4c8, entries=150, sequenceid=251, filesize=11.9 K 2024-11-17T01:54:02,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/0c7cbbcb37af42adbea8f634f861f073 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/0c7cbbcb37af42adbea8f634f861f073 2024-11-17T01:54:02,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/0c7cbbcb37af42adbea8f634f861f073, entries=150, sequenceid=251, filesize=11.9 K 2024-11-17T01:54:02,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 5e67ace45ed3f9aacb3b2064d2a9351c in 1245ms, sequenceid=251, compaction requested=true 2024-11-17T01:54:02,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:54:02,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:54:02,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:02,324 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:54:02,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:54:02,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:02,324 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:54:02,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:54:02,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:54:02,325 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 58608 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:54:02,325 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/A is initiating minor compaction (all files) 2024-11-17T01:54:02,325 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:54:02,325 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/B is initiating minor compaction (all files) 2024-11-17T01:54:02,325 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/A in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:02,325 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/B in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:02,325 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/6c0a161a4ea94501ad2092ed7a02a7d3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/0ec8ce82a8cb460eb254d39901fa6b62, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f888be73f069469aabbfb072e2de6414, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/df41d6cb9d454bc3a5b46db8752ce4c8] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=47.9 K 2024-11-17T01:54:02,325 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/dea26d96b03f4240a0f817ff6ef80d06, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/963875ab5ab84cf887817dadbc812106, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/424be89a8ecf4479a70e60c10968bfca, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/fb098d231c8a426fab345ddfe8cc9016] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=57.2 K 2024-11-17T01:54:02,325 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c0a161a4ea94501ad2092ed7a02a7d3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731808437319 2024-11-17T01:54:02,325 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting dea26d96b03f4240a0f817ff6ef80d06, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731808437319 2024-11-17T01:54:02,325 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ec8ce82a8cb460eb254d39901fa6b62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731808437687 2024-11-17T01:54:02,325 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 963875ab5ab84cf887817dadbc812106, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731808437687 2024-11-17T01:54:02,326 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting f888be73f069469aabbfb072e2de6414, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731808438859 2024-11-17T01:54:02,326 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 424be89a8ecf4479a70e60c10968bfca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731808438859 2024-11-17T01:54:02,326 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting df41d6cb9d454bc3a5b46db8752ce4c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731808441075 2024-11-17T01:54:02,326 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb098d231c8a426fab345ddfe8cc9016, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731808441072 2024-11-17T01:54:02,334 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#B#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:02,335 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/f6e051460a5c4a54a0e63059567ca481 is 50, key is test_row_0/B:col10/1731808441078/Put/seqid=0 2024-11-17T01:54:02,335 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#A#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:02,336 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/389da4f144994659ab2977b133c2cd92 is 50, key is test_row_0/A:col10/1731808441078/Put/seqid=0 2024-11-17T01:54:02,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742323_1499 (size=12731) 2024-11-17T01:54:02,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742322_1498 (size=12731) 2024-11-17T01:54:02,356 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/f6e051460a5c4a54a0e63059567ca481 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f6e051460a5c4a54a0e63059567ca481 2024-11-17T01:54:02,359 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/B of 5e67ace45ed3f9aacb3b2064d2a9351c into f6e051460a5c4a54a0e63059567ca481(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:02,360 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:54:02,360 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/B, priority=12, startTime=1731808442324; duration=0sec 2024-11-17T01:54:02,360 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:54:02,360 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:B 2024-11-17T01:54:02,360 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:54:02,361 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:54:02,361 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/C is initiating minor compaction (all files) 2024-11-17T01:54:02,361 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/C in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:02,361 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/cabe72e632f142a682f3d60eb25e0556, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/fdfd09a1f74a489fa9842cfe9a851c58, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/ab4fed1b0ad74dae9954985d6d75d192, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/0c7cbbcb37af42adbea8f634f861f073] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=47.9 K 2024-11-17T01:54:02,361 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting cabe72e632f142a682f3d60eb25e0556, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731808437319 2024-11-17T01:54:02,361 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting fdfd09a1f74a489fa9842cfe9a851c58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731808437687 2024-11-17T01:54:02,361 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting ab4fed1b0ad74dae9954985d6d75d192, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731808438859 2024-11-17T01:54:02,362 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c7cbbcb37af42adbea8f634f861f073, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731808441075 2024-11-17T01:54:02,368 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#C#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:02,368 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/d32d8f6f34cd49e1bef482b17f1ddab3 is 50, key is test_row_0/C:col10/1731808441078/Put/seqid=0 2024-11-17T01:54:02,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742324_1500 (size=12731) 2024-11-17T01:54:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:54:02,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e67ace45ed3f9aacb3b2064d2a9351c 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-17T01:54:02,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=A 2024-11-17T01:54:02,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:54:02,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=B 2024-11-17T01:54:02,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:54:02,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=C 2024-11-17T01:54:02,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:54:02,381 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/d32d8f6f34cd49e1bef482b17f1ddab3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/d32d8f6f34cd49e1bef482b17f1ddab3 2024-11-17T01:54:02,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/4b2641977cbe43eb8e0ebb4bfdcbe918 is 50, key is test_row_0/A:col10/1731808442378/Put/seqid=0 2024-11-17T01:54:02,386 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/C of 5e67ace45ed3f9aacb3b2064d2a9351c into d32d8f6f34cd49e1bef482b17f1ddab3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:02,386 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:54:02,386 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/C, priority=12, startTime=1731808442324; duration=0sec 2024-11-17T01:54:02,386 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:02,386 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:C 2024-11-17T01:54:02,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742325_1501 (size=14741) 2024-11-17T01:54:02,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T01:54:02,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:02,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:54:02,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:02,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:02,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:02,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:02,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808502416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:02,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:02,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:02,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:02,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:02,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808502426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:02,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:02,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:02,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:02,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:02,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808502534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:02,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:02,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808502534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:02,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T01:54:02,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:02,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:54:02,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:02,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:02,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:02,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:02,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T01:54:02,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:02,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:54:02,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:02,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:02,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:02,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:02,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:02,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808502744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:02,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:02,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:02,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808502744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:02,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:02,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:02,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:02,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:02,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:02,755 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/389da4f144994659ab2977b133c2cd92 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/389da4f144994659ab2977b133c2cd92 2024-11-17T01:54:02,758 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/A of 5e67ace45ed3f9aacb3b2064d2a9351c into 389da4f144994659ab2977b133c2cd92(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:02,758 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:54:02,758 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/A, priority=12, startTime=1731808442324; duration=0sec 2024-11-17T01:54:02,759 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:02,759 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:A 2024-11-17T01:54:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T01:54:02,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/4b2641977cbe43eb8e0ebb4bfdcbe918 2024-11-17T01:54:02,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/624f607520bd4ac3a8333b5198a20aa3 is 50, key is test_row_0/B:col10/1731808442378/Put/seqid=0 2024-11-17T01:54:02,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742326_1502 (size=12301) 2024-11-17T01:54:02,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/624f607520bd4ac3a8333b5198a20aa3 2024-11-17T01:54:02,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/71093efec090450c90979e9f14690403 is 50, key is test_row_0/C:col10/1731808442378/Put/seqid=0 2024-11-17T01:54:02,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742327_1503 (size=12301) 2024-11-17T01:54:02,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T01:54:02,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:02,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:54:02,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:02,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:02,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:03,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T01:54:03,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:03,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:54:03,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:03,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:03,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:03,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808503054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:03,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808503055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:03,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:03,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T01:54:03,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:03,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:54:03,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:03,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:03,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:03,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/71093efec090450c90979e9f14690403 2024-11-17T01:54:03,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/4b2641977cbe43eb8e0ebb4bfdcbe918 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/4b2641977cbe43eb8e0ebb4bfdcbe918 2024-11-17T01:54:03,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/4b2641977cbe43eb8e0ebb4bfdcbe918, entries=200, sequenceid=287, filesize=14.4 K 2024-11-17T01:54:03,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/624f607520bd4ac3a8333b5198a20aa3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/624f607520bd4ac3a8333b5198a20aa3 2024-11-17T01:54:03,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/624f607520bd4ac3a8333b5198a20aa3, entries=150, sequenceid=287, filesize=12.0 K 2024-11-17T01:54:03,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/71093efec090450c90979e9f14690403 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/71093efec090450c90979e9f14690403 2024-11-17T01:54:03,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/71093efec090450c90979e9f14690403, entries=150, sequenceid=287, filesize=12.0 K 2024-11-17T01:54:03,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for 5e67ace45ed3f9aacb3b2064d2a9351c in 844ms, sequenceid=287, compaction requested=false 2024-11-17T01:54:03,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:54:03,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T01:54:03,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:03,307 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 5e67ace45ed3f9aacb3b2064d2a9351c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T01:54:03,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=A 2024-11-17T01:54:03,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:03,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=B 2024-11-17T01:54:03,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:03,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=C 2024-11-17T01:54:03,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:03,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/5986ee0cbe91485a9832266584270ef8 is 50, key is test_row_0/A:col10/1731808442404/Put/seqid=0 2024-11-17T01:54:03,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742328_1504 (size=12301) 2024-11-17T01:54:03,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:54:03,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:54:03,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808443355, max timestamp=1731808443364Num uniques 859; 2024-11-17T01:54:03,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:54:03,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808443355, max timestamp=1731808443364Num uniques 859; 2024-11-17T01:54:03,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:54:03,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731808443355, max timestamp=1731808443364Num uniques 859; 2024-11-17T01:54:03,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T01:54:03,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808503414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:03,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808503420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:03,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:03,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808503421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:03,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:03,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808503534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808503534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:03,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808503544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:03,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:03,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808503574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808503575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:03,715 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/5986ee0cbe91485a9832266584270ef8 2024-11-17T01:54:03,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/12d2df4b279f4129a0959cdc31a617c4 is 50, key is test_row_0/B:col10/1731808442404/Put/seqid=0 2024-11-17T01:54:03,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742329_1505 (size=12301) 2024-11-17T01:54:03,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808503744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808503744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:03,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808503754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T01:54:04,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808504064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:04,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808504064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:04,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808504064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:04,124 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/12d2df4b279f4129a0959cdc31a617c4 2024-11-17T01:54:04,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/6ede9e50ec0b4fcf81a1570d18672fa2 is 50, key is test_row_0/C:col10/1731808442404/Put/seqid=0 2024-11-17T01:54:04,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742330_1506 (size=12301) 2024-11-17T01:54:04,533 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/6ede9e50ec0b4fcf81a1570d18672fa2 2024-11-17T01:54:04,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/5986ee0cbe91485a9832266584270ef8 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/5986ee0cbe91485a9832266584270ef8 2024-11-17T01:54:04,538 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/5986ee0cbe91485a9832266584270ef8, entries=150, sequenceid=299, filesize=12.0 K 2024-11-17T01:54:04,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/12d2df4b279f4129a0959cdc31a617c4 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/12d2df4b279f4129a0959cdc31a617c4 2024-11-17T01:54:04,543 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/12d2df4b279f4129a0959cdc31a617c4, entries=150, sequenceid=299, filesize=12.0 K 2024-11-17T01:54:04,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/6ede9e50ec0b4fcf81a1570d18672fa2 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/6ede9e50ec0b4fcf81a1570d18672fa2 2024-11-17T01:54:04,547 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/6ede9e50ec0b4fcf81a1570d18672fa2, entries=150, sequenceid=299, filesize=12.0 K 2024-11-17T01:54:04,547 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 5e67ace45ed3f9aacb3b2064d2a9351c in 1240ms, sequenceid=299, compaction requested=true 2024-11-17T01:54:04,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:54:04,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:04,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-17T01:54:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-17T01:54:04,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-17T01:54:04,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9220 sec 2024-11-17T01:54:04,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.9300 sec 2024-11-17T01:54:04,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:54:04,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e67ace45ed3f9aacb3b2064d2a9351c 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-17T01:54:04,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=A 2024-11-17T01:54:04,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T01:54:04,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=B 2024-11-17T01:54:04,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T01:54:04,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=C 2024-11-17T01:54:04,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T01:54:04,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808504592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:04,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808504593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:04,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808504594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:04,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808504598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808504599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:04,605 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_2' in 5e67ace45ed3f9aacb3b2064d2a9351c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:04,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/786117b448bd4a0fa5e80a6af4fda01f is 50, key is test_row_0/A:col10/1731808444582/Put/seqid=0 2024-11-17T01:54:04,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742331_1507 (size=14741) 2024-11-17T01:54:04,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808504704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:04,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808504704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:04,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:04,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808504714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:04,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808504714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:04,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:04,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808504914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:04,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808504914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:04,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:04,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808504934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:04,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:04,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:04,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808504934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:04,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:04,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:05,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/786117b448bd4a0fa5e80a6af4fda01f 2024-11-17T01:54:05,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/d37a77c2502f4219864fb2bd39ce8d7b is 50, key is test_row_0/B:col10/1731808444582/Put/seqid=0 2024-11-17T01:54:05,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742332_1508 (size=12301) 2024-11-17T01:54:05,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808505224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:05,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808505224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:05,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808505254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:05,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:05,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:05,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:05,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808505254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:05,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/d37a77c2502f4219864fb2bd39ce8d7b 2024-11-17T01:54:05,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/b76c2ac636e946078d065dc869baf92b is 50, key is test_row_0/C:col10/1731808444582/Put/seqid=0 2024-11-17T01:54:05,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742333_1509 (size=12301) 2024-11-17T01:54:05,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808505734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:05,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808505734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:05,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:05,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:05,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:05,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:05,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:05,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:05,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808505774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:05,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:05,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:05,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808505774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:05,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:05,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:05,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:05,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:05,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T01:54:05,794 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:54:05,794 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:54:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-17T01:54:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T01:54:05,796 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:54:05,796 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:54:05,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:54:05,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/b76c2ac636e946078d065dc869baf92b 2024-11-17T01:54:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/786117b448bd4a0fa5e80a6af4fda01f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/786117b448bd4a0fa5e80a6af4fda01f 2024-11-17T01:54:05,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/786117b448bd4a0fa5e80a6af4fda01f, entries=200, sequenceid=337, filesize=14.4 K 2024-11-17T01:54:05,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/d37a77c2502f4219864fb2bd39ce8d7b as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/d37a77c2502f4219864fb2bd39ce8d7b 2024-11-17T01:54:05,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/d37a77c2502f4219864fb2bd39ce8d7b, entries=150, sequenceid=337, filesize=12.0 K 2024-11-17T01:54:05,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/b76c2ac636e946078d065dc869baf92b as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/b76c2ac636e946078d065dc869baf92b 2024-11-17T01:54:05,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/b76c2ac636e946078d065dc869baf92b, entries=150, sequenceid=337, filesize=12.0 K 2024-11-17T01:54:05,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 5e67ace45ed3f9aacb3b2064d2a9351c in 1287ms, sequenceid=337, compaction requested=true 2024-11-17T01:54:05,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:54:05,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:54:05,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:05,870 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:54:05,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:54:05,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:05,871 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:54:05,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:54:05,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:54:05,871 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:54:05,871 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54514 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:54:05,871 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/A is initiating minor compaction (all files) 2024-11-17T01:54:05,871 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/B is initiating minor compaction (all files) 2024-11-17T01:54:05,871 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/A in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:05,871 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/B in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:05,872 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/389da4f144994659ab2977b133c2cd92, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/4b2641977cbe43eb8e0ebb4bfdcbe918, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/5986ee0cbe91485a9832266584270ef8, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/786117b448bd4a0fa5e80a6af4fda01f] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=53.2 K 2024-11-17T01:54:05,872 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f6e051460a5c4a54a0e63059567ca481, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/624f607520bd4ac3a8333b5198a20aa3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/12d2df4b279f4129a0959cdc31a617c4, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/d37a77c2502f4219864fb2bd39ce8d7b] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=48.5 K 2024-11-17T01:54:05,872 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 389da4f144994659ab2977b133c2cd92, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731808441075 2024-11-17T01:54:05,872 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6e051460a5c4a54a0e63059567ca481, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731808441075 2024-11-17T01:54:05,872 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 624f607520bd4ac3a8333b5198a20aa3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731808441144 2024-11-17T01:54:05,872 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b2641977cbe43eb8e0ebb4bfdcbe918, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731808441144 2024-11-17T01:54:05,872 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12d2df4b279f4129a0959cdc31a617c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731808442404 2024-11-17T01:54:05,872 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 5986ee0cbe91485a9832266584270ef8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731808442404 2024-11-17T01:54:05,872 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting d37a77c2502f4219864fb2bd39ce8d7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731808443418 2024-11-17T01:54:05,872 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 786117b448bd4a0fa5e80a6af4fda01f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731808443403 2024-11-17T01:54:05,879 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#A#compaction#433 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:05,879 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#B#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:05,880 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/79d39bf42bd947d895ad1ef311ef5ed7 is 50, key is test_row_0/A:col10/1731808444582/Put/seqid=0 2024-11-17T01:54:05,880 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/f2a4b900a8cc42a0b6a890cb4dc93a37 is 50, key is test_row_0/B:col10/1731808444582/Put/seqid=0 2024-11-17T01:54:05,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742334_1510 (size=13017) 2024-11-17T01:54:05,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742335_1511 (size=13017) 2024-11-17T01:54:05,892 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/79d39bf42bd947d895ad1ef311ef5ed7 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/79d39bf42bd947d895ad1ef311ef5ed7 2024-11-17T01:54:05,893 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/f2a4b900a8cc42a0b6a890cb4dc93a37 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f2a4b900a8cc42a0b6a890cb4dc93a37 2024-11-17T01:54:05,896 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/A of 5e67ace45ed3f9aacb3b2064d2a9351c into 79d39bf42bd947d895ad1ef311ef5ed7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:05,896 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/B of 5e67ace45ed3f9aacb3b2064d2a9351c into f2a4b900a8cc42a0b6a890cb4dc93a37(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:05,896 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:54:05,896 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:54:05,896 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/A, priority=12, startTime=1731808445870; duration=0sec 2024-11-17T01:54:05,896 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/B, priority=12, startTime=1731808445870; duration=0sec 2024-11-17T01:54:05,897 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:54:05,897 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:B 2024-11-17T01:54:05,897 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:05,897 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:A 2024-11-17T01:54:05,897 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T01:54:05,898 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T01:54:05,898 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/C is initiating minor compaction (all files) 2024-11-17T01:54:05,898 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/C in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:05,898 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/d32d8f6f34cd49e1bef482b17f1ddab3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/71093efec090450c90979e9f14690403, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/6ede9e50ec0b4fcf81a1570d18672fa2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/b76c2ac636e946078d065dc869baf92b] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=48.5 K 2024-11-17T01:54:05,898 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting d32d8f6f34cd49e1bef482b17f1ddab3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731808441075 2024-11-17T01:54:05,899 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71093efec090450c90979e9f14690403, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731808441144 2024-11-17T01:54:05,899 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ede9e50ec0b4fcf81a1570d18672fa2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731808442404 2024-11-17T01:54:05,899 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting b76c2ac636e946078d065dc869baf92b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731808443418 2024-11-17T01:54:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T01:54:05,906 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#C#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:05,907 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/0f49789e2b4c4ffbacf047b3646ee250 is 50, key is test_row_0/C:col10/1731808444582/Put/seqid=0 2024-11-17T01:54:05,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742336_1512 (size=13017) 2024-11-17T01:54:05,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T01:54:05,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:05,949 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 5e67ace45ed3f9aacb3b2064d2a9351c 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-17T01:54:05,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=A 2024-11-17T01:54:05,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:05,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=B 2024-11-17T01:54:05,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:05,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=C 2024-11-17T01:54:05,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:05,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/322a38d589db403f9292ddc1de331b23 is 50, key is test_row_1/A:col10/1731808444598/Put/seqid=0 2024-11-17T01:54:05,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742337_1513 (size=9857) 2024-11-17T01:54:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T01:54:06,320 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/0f49789e2b4c4ffbacf047b3646ee250 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/0f49789e2b4c4ffbacf047b3646ee250 2024-11-17T01:54:06,323 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/C of 5e67ace45ed3f9aacb3b2064d2a9351c into 0f49789e2b4c4ffbacf047b3646ee250(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:06,323 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:54:06,323 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/C, priority=12, startTime=1731808445871; duration=0sec 2024-11-17T01:54:06,324 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:06,324 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:C 2024-11-17T01:54:06,360 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/322a38d589db403f9292ddc1de331b23 2024-11-17T01:54:06,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/bd3d7aa4cb6e4656b2c1626c2b57936e is 50, key is test_row_1/B:col10/1731808444598/Put/seqid=0 2024-11-17T01:54:06,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742338_1514 (size=9857) 2024-11-17T01:54:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T01:54:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:54:06,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. as already flushing 2024-11-17T01:54:06,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808446687, max timestamp=1731808446709Num uniques 859; 2024-11-17T01:54:06,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808446687, max timestamp=1731808446709Num uniques 859; 2024-11-17T01:54:06,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808446687, max timestamp=1731808446709Num uniques 859; 2024-11-17T01:54:06,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:54:06,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:54:06,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:54:06,768 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/bd3d7aa4cb6e4656b2c1626c2b57936e 2024-11-17T01:54:06,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/4562e311f03f416b83b14d2a938b9f20 is 50, key is test_row_1/C:col10/1731808444598/Put/seqid=0 2024-11-17T01:54:06,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742339_1515 (size=9857) 2024-11-17T01:54:06,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808506770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:06,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:06,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:06,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:06,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808506779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:06,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808506779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:06,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57082 deadline: 1731808506794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:06,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57058 deadline: 1731808506794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:06,800 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in 5e67ace45ed3f9aacb3b2064d2a9351c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:06,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:06,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:06,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:06,801 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in 5e67ace45ed3f9aacb3b2064d2a9351c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:06,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:06,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:06,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:06,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:06,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808506884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:06,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:06,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808506884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:06,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808506894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:06,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:06,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:06,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T01:54:07,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808507094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:07,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808507094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:07,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808507104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:07,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:07,176 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/4562e311f03f416b83b14d2a938b9f20 2024-11-17T01:54:07,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/322a38d589db403f9292ddc1de331b23 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/322a38d589db403f9292ddc1de331b23 2024-11-17T01:54:07,181 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/322a38d589db403f9292ddc1de331b23, entries=100, sequenceid=348, filesize=9.6 K 2024-11-17T01:54:07,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/bd3d7aa4cb6e4656b2c1626c2b57936e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/bd3d7aa4cb6e4656b2c1626c2b57936e 2024-11-17T01:54:07,185 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/bd3d7aa4cb6e4656b2c1626c2b57936e, entries=100, sequenceid=348, filesize=9.6 K 2024-11-17T01:54:07,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/4562e311f03f416b83b14d2a938b9f20 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/4562e311f03f416b83b14d2a938b9f20 2024-11-17T01:54:07,188 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/4562e311f03f416b83b14d2a938b9f20, entries=100, sequenceid=348, filesize=9.6 K 2024-11-17T01:54:07,188 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 5e67ace45ed3f9aacb3b2064d2a9351c in 1240ms, sequenceid=348, compaction requested=false 2024-11-17T01:54:07,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:54:07,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:07,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-17T01:54:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-17T01:54:07,190 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-17T01:54:07,190 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3930 sec 2024-11-17T01:54:07,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.3970 sec 2024-11-17T01:54:07,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e67ace45ed3f9aacb3b2064d2a9351c 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-17T01:54:07,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=A 2024-11-17T01:54:07,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T01:54:07,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=B 2024-11-17T01:54:07,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T01:54:07,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=C 2024-11-17T01:54:07,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T01:54:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:54:07,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/7c08caa3312d47db955f18f2f752fbd8 is 50, key is test_row_0/A:col10/1731808446778/Put/seqid=0 2024-11-17T01:54:07,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742340_1516 (size=14741) 2024-11-17T01:54:07,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/7c08caa3312d47db955f18f2f752fbd8 2024-11-17T01:54:07,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/ba972d81f39e481da111eb31df21d664 is 50, key is test_row_0/B:col10/1731808446778/Put/seqid=0 2024-11-17T01:54:07,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808507427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:07,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:07,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808507433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:07,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:07,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742341_1517 (size=12301) 2024-11-17T01:54:07,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808507434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:07,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:07,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808507544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808507544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:07,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808507554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:07,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:07,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57048 deadline: 1731808507754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:07,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:07,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57052 deadline: 1731808507754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:07,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:07,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57032 deadline: 1731808507764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 , the old value is region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=2 because the exception is null or not the one we care about 2024-11-17T01:54:07,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/ba972d81f39e481da111eb31df21d664 2024-11-17T01:54:07,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/4195efd15eb44d0b8179efe60e2bea92 is 50, key is test_row_0/C:col10/1731808446778/Put/seqid=0 2024-11-17T01:54:07,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742342_1518 (size=12301) 2024-11-17T01:54:07,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/4195efd15eb44d0b8179efe60e2bea92 2024-11-17T01:54:07,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/7c08caa3312d47db955f18f2f752fbd8 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/7c08caa3312d47db955f18f2f752fbd8 2024-11-17T01:54:07,852 INFO [Thread-2018 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2018. 2024-11-17T01:54:07,852 DEBUG [Thread-2018 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:54:07,852 DEBUG [Thread-2018 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:07,852 DEBUG [Thread-2018 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:07,852 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:07,855 INFO [Thread-2016 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2016. 2024-11-17T01:54:07,855 DEBUG [Thread-2016 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:54:07,855 DEBUG [Thread-2016 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:07,855 DEBUG [Thread-2016 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:07,855 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:07,856 INFO [Thread-2012 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2012. 2024-11-17T01:54:07,856 DEBUG [Thread-2012 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:54:07,856 DEBUG [Thread-2012 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:07,856 DEBUG [Thread-2012 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:07,856 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:07,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/7c08caa3312d47db955f18f2f752fbd8, entries=200, sequenceid=387, filesize=14.4 K 2024-11-17T01:54:07,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/ba972d81f39e481da111eb31df21d664 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/ba972d81f39e481da111eb31df21d664 2024-11-17T01:54:07,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/ba972d81f39e481da111eb31df21d664, entries=150, sequenceid=387, filesize=12.0 K 2024-11-17T01:54:07,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/4195efd15eb44d0b8179efe60e2bea92 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/4195efd15eb44d0b8179efe60e2bea92 2024-11-17T01:54:07,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/4195efd15eb44d0b8179efe60e2bea92, entries=150, sequenceid=387, filesize=12.0 K 2024-11-17T01:54:07,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 5e67ace45ed3f9aacb3b2064d2a9351c in 447ms, sequenceid=387, compaction requested=true 2024-11-17T01:54:07,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:54:07,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:54:07,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:07,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:54:07,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:07,863 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:07,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e67ace45ed3f9aacb3b2064d2a9351c:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:54:07,863 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:07,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:54:07,864 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37615 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:07,864 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:07,864 INFO [Thread-2020 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2020. 2024-11-17T01:54:07,864 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/A is initiating minor compaction (all files) 2024-11-17T01:54:07,864 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/B is initiating minor compaction (all files) 2024-11-17T01:54:07,864 DEBUG [Thread-2020 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:54:07,864 DEBUG [Thread-2020 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:07,864 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/B in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:07,864 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/A in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:07,864 DEBUG [Thread-2020 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:07,864 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:07,864 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/79d39bf42bd947d895ad1ef311ef5ed7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/322a38d589db403f9292ddc1de331b23, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/7c08caa3312d47db955f18f2f752fbd8] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=36.7 K 2024-11-17T01:54:07,864 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f2a4b900a8cc42a0b6a890cb4dc93a37, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/bd3d7aa4cb6e4656b2c1626c2b57936e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/ba972d81f39e481da111eb31df21d664] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=34.4 K 2024-11-17T01:54:07,865 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting f2a4b900a8cc42a0b6a890cb4dc93a37, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731808443418 2024-11-17T01:54:07,865 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79d39bf42bd947d895ad1ef311ef5ed7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731808443418 2024-11-17T01:54:07,865 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting bd3d7aa4cb6e4656b2c1626c2b57936e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731808444592 2024-11-17T01:54:07,865 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 322a38d589db403f9292ddc1de331b23, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731808444592 2024-11-17T01:54:07,865 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting ba972d81f39e481da111eb31df21d664, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731808446768 2024-11-17T01:54:07,865 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c08caa3312d47db955f18f2f752fbd8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731808446768 2024-11-17T01:54:07,866 INFO [Thread-2014 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2014. 2024-11-17T01:54:07,867 DEBUG [Thread-2014 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:54:07,867 DEBUG [Thread-2014 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:07,867 DEBUG [Thread-2014 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:07,867 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:07,871 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#B#compaction#443 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:07,871 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#A#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:07,871 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/fc76647868df496394997711c25cb740 is 50, key is test_row_0/B:col10/1731808446778/Put/seqid=0 2024-11-17T01:54:07,871 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/129519b83c074277bd8f10211ac0a5b4 is 50, key is test_row_0/A:col10/1731808446778/Put/seqid=0 2024-11-17T01:54:07,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742343_1519 (size=13119) 2024-11-17T01:54:07,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742344_1520 (size=13119) 2024-11-17T01:54:07,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T01:54:07,954 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:54:08,068 INFO [Thread-2001 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2001. 2024-11-17T01:54:08,068 DEBUG [Thread-2001 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:54:08,068 DEBUG [Thread-2001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:08,069 DEBUG [Thread-2001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:08,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:54:08,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e67ace45ed3f9aacb3b2064d2a9351c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T01:54:08,078 INFO [Thread-2003 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2003. 2024-11-17T01:54:08,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=A 2024-11-17T01:54:08,078 DEBUG [Thread-2003 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:54:08,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:08,078 DEBUG [Thread-2003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:08,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=B 2024-11-17T01:54:08,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:08,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=C 2024-11-17T01:54:08,078 DEBUG [Thread-2003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:08,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:08,078 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:08,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/48516f47ff0f4aa99ebb123263580932 is 50, key is test_row_0/A:col10/1731808448076/Put/seqid=0 2024-11-17T01:54:08,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742345_1521 (size=9857) 2024-11-17T01:54:08,086 INFO [Thread-2007 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2007. 2024-11-17T01:54:08,086 DEBUG [Thread-2007 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:54:08,086 DEBUG [Thread-2007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:08,086 DEBUG [Thread-2007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:08,086 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:08,278 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/fc76647868df496394997711c25cb740 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/fc76647868df496394997711c25cb740 2024-11-17T01:54:08,279 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/129519b83c074277bd8f10211ac0a5b4 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/129519b83c074277bd8f10211ac0a5b4 2024-11-17T01:54:08,282 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/B of 5e67ace45ed3f9aacb3b2064d2a9351c into fc76647868df496394997711c25cb740(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:08,282 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:54:08,282 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/B, priority=13, startTime=1731808447863; duration=0sec 2024-11-17T01:54:08,282 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:54:08,282 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/A of 5e67ace45ed3f9aacb3b2064d2a9351c into 129519b83c074277bd8f10211ac0a5b4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:08,282 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:B 2024-11-17T01:54:08,282 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:54:08,282 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/A, priority=13, startTime=1731808447863; duration=0sec 2024-11-17T01:54:08,282 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:08,282 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:08,282 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:A 2024-11-17T01:54:08,283 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:08,283 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): 5e67ace45ed3f9aacb3b2064d2a9351c/C is initiating minor compaction (all files) 2024-11-17T01:54:08,283 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e67ace45ed3f9aacb3b2064d2a9351c/C in TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:08,283 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/0f49789e2b4c4ffbacf047b3646ee250, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/4562e311f03f416b83b14d2a938b9f20, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/4195efd15eb44d0b8179efe60e2bea92] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp, totalSize=34.4 K 2024-11-17T01:54:08,283 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f49789e2b4c4ffbacf047b3646ee250, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731808443418 2024-11-17T01:54:08,283 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 4562e311f03f416b83b14d2a938b9f20, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731808444592 2024-11-17T01:54:08,284 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 4195efd15eb44d0b8179efe60e2bea92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731808446768 2024-11-17T01:54:08,288 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e67ace45ed3f9aacb3b2064d2a9351c#C#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:08,289 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/e000ab4207cb4379a975271672cc437b is 50, key is test_row_0/C:col10/1731808446778/Put/seqid=0 2024-11-17T01:54:08,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742346_1522 (size=13119) 2024-11-17T01:54:08,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/48516f47ff0f4aa99ebb123263580932 2024-11-17T01:54:08,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/530bb48b00304b18b4588780dd1d1102 is 50, key is test_row_0/B:col10/1731808448076/Put/seqid=0 2024-11-17T01:54:08,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742347_1523 (size=9857) 2024-11-17T01:54:08,702 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/e000ab4207cb4379a975271672cc437b as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/e000ab4207cb4379a975271672cc437b 2024-11-17T01:54:08,707 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e67ace45ed3f9aacb3b2064d2a9351c/C of 5e67ace45ed3f9aacb3b2064d2a9351c into e000ab4207cb4379a975271672cc437b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:08,708 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:54:08,708 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., storeName=5e67ace45ed3f9aacb3b2064d2a9351c/C, priority=13, startTime=1731808447863; duration=0sec 2024-11-17T01:54:08,708 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:08,708 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e67ace45ed3f9aacb3b2064d2a9351c:C 2024-11-17T01:54:08,806 INFO [Thread-2009 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2009. 2024-11-17T01:54:08,806 DEBUG [Thread-2009 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:54:08,806 DEBUG [Thread-2009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:08,806 DEBUG [Thread-2009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:08,806 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:08,827 INFO [Thread-2005 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2005. 2024-11-17T01:54:08,828 DEBUG [Thread-2005 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:54:08,828 DEBUG [Thread-2005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:08,828 DEBUG [Thread-2005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:08,828 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:08,828 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T01:54:08,828 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-17T01:54:08,829 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-17T01:54:08,829 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-17T01:54:08,829 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-17T01:54:08,829 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-17T01:54:08,829 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T01:54:08,829 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T01:54:08,829 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1226 2024-11-17T01:54:08,829 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3678 rows 2024-11-17T01:54:08,829 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1233 2024-11-17T01:54:08,829 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3691 rows 2024-11-17T01:54:08,829 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1220 2024-11-17T01:54:08,829 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3660 rows 2024-11-17T01:54:08,829 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1226 2024-11-17T01:54:08,829 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3673 rows 2024-11-17T01:54:08,830 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1221 2024-11-17T01:54:08,830 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3663 rows 2024-11-17T01:54:08,830 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T01:54:08,830 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:54:08,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:08,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:08,831 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:08,832 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T01:54:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T01:54:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T01:54:08,837 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808448837"}]},"ts":"1731808448837"} 2024-11-17T01:54:08,839 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T01:54:08,839 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T01:54:08,839 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T01:54:08,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e67ace45ed3f9aacb3b2064d2a9351c, UNASSIGN}] 2024-11-17T01:54:08,842 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e67ace45ed3f9aacb3b2064d2a9351c, UNASSIGN 2024-11-17T01:54:08,843 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=5e67ace45ed3f9aacb3b2064d2a9351c, regionState=CLOSING, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:08,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e67ace45ed3f9aacb3b2064d2a9351c, UNASSIGN because future has completed 2024-11-17T01:54:08,845 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T01:54:08,845 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173}] 2024-11-17T01:54:08,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/530bb48b00304b18b4588780dd1d1102 2024-11-17T01:54:08,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/a241b1f48f47456db6d4592f289b1dc2 is 50, key is test_row_0/C:col10/1731808448076/Put/seqid=0 2024-11-17T01:54:08,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742348_1524 (size=9857) 2024-11-17T01:54:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T01:54:08,998 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:54:08,998 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T01:54:08,998 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 5e67ace45ed3f9aacb3b2064d2a9351c, disabling compactions & flushes 2024-11-17T01:54:08,998 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T01:54:09,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/a241b1f48f47456db6d4592f289b1dc2 2024-11-17T01:54:09,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/48516f47ff0f4aa99ebb123263580932 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/48516f47ff0f4aa99ebb123263580932 2024-11-17T01:54:09,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/48516f47ff0f4aa99ebb123263580932, entries=100, sequenceid=397, filesize=9.6 K 2024-11-17T01:54:09,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/530bb48b00304b18b4588780dd1d1102 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/530bb48b00304b18b4588780dd1d1102 2024-11-17T01:54:09,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/530bb48b00304b18b4588780dd1d1102, entries=100, sequenceid=397, filesize=9.6 K 2024-11-17T01:54:09,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/a241b1f48f47456db6d4592f289b1dc2 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/a241b1f48f47456db6d4592f289b1dc2 2024-11-17T01:54:09,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/a241b1f48f47456db6d4592f289b1dc2, entries=100, sequenceid=397, filesize=9.6 K 2024-11-17T01:54:09,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=20.13 KB/20610 for 5e67ace45ed3f9aacb3b2064d2a9351c in 1260ms, sequenceid=397, compaction requested=false 2024-11-17T01:54:09,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e67ace45ed3f9aacb3b2064d2a9351c: 2024-11-17T01:54:09,338 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:09,338 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:09,338 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. after waiting 0 ms 2024-11-17T01:54:09,338 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:09,338 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 5e67ace45ed3f9aacb3b2064d2a9351c 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-17T01:54:09,338 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=A 2024-11-17T01:54:09,338 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:09,339 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=B 2024-11-17T01:54:09,339 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:09,339 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5e67ace45ed3f9aacb3b2064d2a9351c, store=C 2024-11-17T01:54:09,339 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:09,343 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/d0323f76f9f44f969bfd50fd70380bfb is 50, key is test_row_0/A:col10/1731808448825/Put/seqid=0 2024-11-17T01:54:09,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742349_1525 (size=12301) 2024-11-17T01:54:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T01:54:09,749 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/d0323f76f9f44f969bfd50fd70380bfb 2024-11-17T01:54:09,802 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/06af3c4800df42f2b9dd132885476b76 is 50, key is test_row_0/B:col10/1731808448825/Put/seqid=0 2024-11-17T01:54:09,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742350_1526 (size=12301) 2024-11-17T01:54:09,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T01:54:10,207 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/06af3c4800df42f2b9dd132885476b76 2024-11-17T01:54:10,217 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/62f66185c21d435382730274a91179a0 is 50, key is test_row_0/C:col10/1731808448825/Put/seqid=0 2024-11-17T01:54:10,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742351_1527 (size=12301) 2024-11-17T01:54:10,621 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/62f66185c21d435382730274a91179a0 2024-11-17T01:54:10,630 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/A/d0323f76f9f44f969bfd50fd70380bfb as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/d0323f76f9f44f969bfd50fd70380bfb 2024-11-17T01:54:10,635 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/d0323f76f9f44f969bfd50fd70380bfb, entries=150, sequenceid=406, filesize=12.0 K 2024-11-17T01:54:10,636 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/B/06af3c4800df42f2b9dd132885476b76 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/06af3c4800df42f2b9dd132885476b76 2024-11-17T01:54:10,641 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/06af3c4800df42f2b9dd132885476b76, entries=150, sequenceid=406, filesize=12.0 K 2024-11-17T01:54:10,642 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/.tmp/C/62f66185c21d435382730274a91179a0 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/62f66185c21d435382730274a91179a0 2024-11-17T01:54:10,647 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/62f66185c21d435382730274a91179a0, entries=150, sequenceid=406, filesize=12.0 K 2024-11-17T01:54:10,648 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 5e67ace45ed3f9aacb3b2064d2a9351c in 1310ms, sequenceid=406, compaction requested=true 2024-11-17T01:54:10,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/d8290f7308a744009177a55b4c1f2390, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/0cb882ec42a748cc95fd137ec6800796, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/321b7c7a99a84c9db512eb25223fb296, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/cd780a01f0994f5782bc7b2f229128bc, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/64edfd2a4eda4cc8b22f005161848dcc, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/065292d58f484f4fbf0b74690c1f3e94, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/7ac7f5be4144437c9912149c070d14ee, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/c07b8a4a48c6480098c6ccee4fd26283, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/925e3d65a16a4680b0affe0d20ebd2f5, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/e83342ec7cd84edaa3866d474934c8c4, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/1aa3bada11bc47d2aeb4e1206ed77a69, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/4accbf13848a41c39295c8e05aa85b48, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/1d24c84cf91844939750abbca603dc6f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/dea26d96b03f4240a0f817ff6ef80d06, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/963875ab5ab84cf887817dadbc812106, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/424be89a8ecf4479a70e60c10968bfca, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/fb098d231c8a426fab345ddfe8cc9016, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/389da4f144994659ab2977b133c2cd92, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/4b2641977cbe43eb8e0ebb4bfdcbe918, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/5986ee0cbe91485a9832266584270ef8, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/786117b448bd4a0fa5e80a6af4fda01f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/79d39bf42bd947d895ad1ef311ef5ed7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/322a38d589db403f9292ddc1de331b23, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/7c08caa3312d47db955f18f2f752fbd8] to archive 2024-11-17T01:54:10,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T01:54:10,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/d8290f7308a744009177a55b4c1f2390 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/d8290f7308a744009177a55b4c1f2390 2024-11-17T01:54:10,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/0cb882ec42a748cc95fd137ec6800796 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/0cb882ec42a748cc95fd137ec6800796 2024-11-17T01:54:10,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/321b7c7a99a84c9db512eb25223fb296 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/321b7c7a99a84c9db512eb25223fb296 2024-11-17T01:54:10,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/cd780a01f0994f5782bc7b2f229128bc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/cd780a01f0994f5782bc7b2f229128bc 2024-11-17T01:54:10,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/64edfd2a4eda4cc8b22f005161848dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/64edfd2a4eda4cc8b22f005161848dcc 2024-11-17T01:54:10,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/065292d58f484f4fbf0b74690c1f3e94 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/065292d58f484f4fbf0b74690c1f3e94 2024-11-17T01:54:10,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/7ac7f5be4144437c9912149c070d14ee to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/7ac7f5be4144437c9912149c070d14ee 2024-11-17T01:54:10,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/c07b8a4a48c6480098c6ccee4fd26283 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/c07b8a4a48c6480098c6ccee4fd26283 2024-11-17T01:54:10,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/925e3d65a16a4680b0affe0d20ebd2f5 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/925e3d65a16a4680b0affe0d20ebd2f5 2024-11-17T01:54:10,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/e83342ec7cd84edaa3866d474934c8c4 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/e83342ec7cd84edaa3866d474934c8c4 2024-11-17T01:54:10,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/1aa3bada11bc47d2aeb4e1206ed77a69 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/1aa3bada11bc47d2aeb4e1206ed77a69 2024-11-17T01:54:10,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/4accbf13848a41c39295c8e05aa85b48 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/4accbf13848a41c39295c8e05aa85b48 2024-11-17T01:54:10,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/1d24c84cf91844939750abbca603dc6f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/1d24c84cf91844939750abbca603dc6f 2024-11-17T01:54:10,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/dea26d96b03f4240a0f817ff6ef80d06 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/dea26d96b03f4240a0f817ff6ef80d06 2024-11-17T01:54:10,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/963875ab5ab84cf887817dadbc812106 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/963875ab5ab84cf887817dadbc812106 2024-11-17T01:54:10,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/424be89a8ecf4479a70e60c10968bfca to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/424be89a8ecf4479a70e60c10968bfca 2024-11-17T01:54:10,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/fb098d231c8a426fab345ddfe8cc9016 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/fb098d231c8a426fab345ddfe8cc9016 2024-11-17T01:54:10,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/389da4f144994659ab2977b133c2cd92 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/389da4f144994659ab2977b133c2cd92 2024-11-17T01:54:10,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/4b2641977cbe43eb8e0ebb4bfdcbe918 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/4b2641977cbe43eb8e0ebb4bfdcbe918 2024-11-17T01:54:10,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/5986ee0cbe91485a9832266584270ef8 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/5986ee0cbe91485a9832266584270ef8 2024-11-17T01:54:10,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/786117b448bd4a0fa5e80a6af4fda01f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/786117b448bd4a0fa5e80a6af4fda01f 2024-11-17T01:54:10,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/79d39bf42bd947d895ad1ef311ef5ed7 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/79d39bf42bd947d895ad1ef311ef5ed7 2024-11-17T01:54:10,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/322a38d589db403f9292ddc1de331b23 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/322a38d589db403f9292ddc1de331b23 2024-11-17T01:54:10,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/7c08caa3312d47db955f18f2f752fbd8 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/7c08caa3312d47db955f18f2f752fbd8 2024-11-17T01:54:10,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/df4d9522e1774797abfb62ec94101eea, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/4fd1682cc4f04529ad6db54770213874, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/4c743bbec11942a88855d8e996487503, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/5966f7616dab418dba2fb7e0077832e6, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/188cd87687444e8491f67a3c2c78f9e4, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/aa390df369d34b9a99685e93d8bdd567, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f337b7da9bd84578ad0320036f9d235c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/14e50ca752914036b3847d44a6b4ee5a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f8077d8d34bb47a7bddddcbbbbcd5694, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/47253a3c8309487697230b44ef1a30d9, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/0da5be46049148b8a67164e04d14a1a6, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/b7025af553ab4635b540b73d49d513b3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/6c0a161a4ea94501ad2092ed7a02a7d3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/9ad39733030e4690943ba2ab03a51593, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/0ec8ce82a8cb460eb254d39901fa6b62, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f888be73f069469aabbfb072e2de6414, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f6e051460a5c4a54a0e63059567ca481, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/df41d6cb9d454bc3a5b46db8752ce4c8, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/624f607520bd4ac3a8333b5198a20aa3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/12d2df4b279f4129a0959cdc31a617c4, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f2a4b900a8cc42a0b6a890cb4dc93a37, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/d37a77c2502f4219864fb2bd39ce8d7b, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/bd3d7aa4cb6e4656b2c1626c2b57936e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/ba972d81f39e481da111eb31df21d664] to archive 2024-11-17T01:54:10,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T01:54:10,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/df4d9522e1774797abfb62ec94101eea to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/df4d9522e1774797abfb62ec94101eea 2024-11-17T01:54:10,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/4fd1682cc4f04529ad6db54770213874 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/4fd1682cc4f04529ad6db54770213874 2024-11-17T01:54:10,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/4c743bbec11942a88855d8e996487503 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/4c743bbec11942a88855d8e996487503 2024-11-17T01:54:10,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/5966f7616dab418dba2fb7e0077832e6 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/5966f7616dab418dba2fb7e0077832e6 2024-11-17T01:54:10,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/188cd87687444e8491f67a3c2c78f9e4 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/188cd87687444e8491f67a3c2c78f9e4 2024-11-17T01:54:10,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/aa390df369d34b9a99685e93d8bdd567 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/aa390df369d34b9a99685e93d8bdd567 2024-11-17T01:54:10,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f337b7da9bd84578ad0320036f9d235c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f337b7da9bd84578ad0320036f9d235c 2024-11-17T01:54:10,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/14e50ca752914036b3847d44a6b4ee5a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/14e50ca752914036b3847d44a6b4ee5a 2024-11-17T01:54:10,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f8077d8d34bb47a7bddddcbbbbcd5694 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f8077d8d34bb47a7bddddcbbbbcd5694 2024-11-17T01:54:10,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/47253a3c8309487697230b44ef1a30d9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/47253a3c8309487697230b44ef1a30d9 2024-11-17T01:54:10,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/0da5be46049148b8a67164e04d14a1a6 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/0da5be46049148b8a67164e04d14a1a6 2024-11-17T01:54:10,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/b7025af553ab4635b540b73d49d513b3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/b7025af553ab4635b540b73d49d513b3 2024-11-17T01:54:10,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/6c0a161a4ea94501ad2092ed7a02a7d3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/6c0a161a4ea94501ad2092ed7a02a7d3 2024-11-17T01:54:10,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/9ad39733030e4690943ba2ab03a51593 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/9ad39733030e4690943ba2ab03a51593 2024-11-17T01:54:10,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/0ec8ce82a8cb460eb254d39901fa6b62 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/0ec8ce82a8cb460eb254d39901fa6b62 2024-11-17T01:54:10,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f888be73f069469aabbfb072e2de6414 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f888be73f069469aabbfb072e2de6414 2024-11-17T01:54:10,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f6e051460a5c4a54a0e63059567ca481 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f6e051460a5c4a54a0e63059567ca481 2024-11-17T01:54:10,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/df41d6cb9d454bc3a5b46db8752ce4c8 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/df41d6cb9d454bc3a5b46db8752ce4c8 2024-11-17T01:54:10,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/624f607520bd4ac3a8333b5198a20aa3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/624f607520bd4ac3a8333b5198a20aa3 2024-11-17T01:54:10,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/12d2df4b279f4129a0959cdc31a617c4 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/12d2df4b279f4129a0959cdc31a617c4 2024-11-17T01:54:10,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f2a4b900a8cc42a0b6a890cb4dc93a37 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/f2a4b900a8cc42a0b6a890cb4dc93a37 2024-11-17T01:54:10,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/d37a77c2502f4219864fb2bd39ce8d7b to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/d37a77c2502f4219864fb2bd39ce8d7b 2024-11-17T01:54:10,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/bd3d7aa4cb6e4656b2c1626c2b57936e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/bd3d7aa4cb6e4656b2c1626c2b57936e 2024-11-17T01:54:10,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/ba972d81f39e481da111eb31df21d664 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/ba972d81f39e481da111eb31df21d664 2024-11-17T01:54:10,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/fb7359e4441544568e68fbb8dc934931, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/be1f431852c344e7a0b94d79ea12cdac, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/cec162b18d1d43b0b7b7089c45984207, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/1defd09408be4ee5b9e4e5a5ac8d065d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/4eade3276295457ea696e2007f8ce32e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/aa95cf3061024a529f122d0149b90e7f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/958485e84cb041669bb3606ac06f4d4e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/ed6bdf158add41129b0a68a60353495d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/16826ff27aa64d21bbaee7003c8d45c2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/d60b336b277f406e8d182d1e333e928e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/74a2445220ac4b93966f223d2123cb7b, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/980205ee49c14967b0741635219d7bf9, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/cabe72e632f142a682f3d60eb25e0556, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/d3e2c8cf94904ce58e30070151fa6e24, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/fdfd09a1f74a489fa9842cfe9a851c58, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/ab4fed1b0ad74dae9954985d6d75d192, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/d32d8f6f34cd49e1bef482b17f1ddab3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/0c7cbbcb37af42adbea8f634f861f073, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/71093efec090450c90979e9f14690403, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/6ede9e50ec0b4fcf81a1570d18672fa2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/0f49789e2b4c4ffbacf047b3646ee250, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/b76c2ac636e946078d065dc869baf92b, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/4562e311f03f416b83b14d2a938b9f20, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/4195efd15eb44d0b8179efe60e2bea92] to archive 2024-11-17T01:54:10,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T01:54:10,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/fb7359e4441544568e68fbb8dc934931 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/fb7359e4441544568e68fbb8dc934931 2024-11-17T01:54:10,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/be1f431852c344e7a0b94d79ea12cdac to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/be1f431852c344e7a0b94d79ea12cdac 2024-11-17T01:54:10,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/cec162b18d1d43b0b7b7089c45984207 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/cec162b18d1d43b0b7b7089c45984207 2024-11-17T01:54:10,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/1defd09408be4ee5b9e4e5a5ac8d065d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/1defd09408be4ee5b9e4e5a5ac8d065d 2024-11-17T01:54:10,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/4eade3276295457ea696e2007f8ce32e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/4eade3276295457ea696e2007f8ce32e 2024-11-17T01:54:10,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/aa95cf3061024a529f122d0149b90e7f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/aa95cf3061024a529f122d0149b90e7f 2024-11-17T01:54:10,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/958485e84cb041669bb3606ac06f4d4e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/958485e84cb041669bb3606ac06f4d4e 2024-11-17T01:54:10,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/ed6bdf158add41129b0a68a60353495d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/ed6bdf158add41129b0a68a60353495d 2024-11-17T01:54:10,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/16826ff27aa64d21bbaee7003c8d45c2 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/16826ff27aa64d21bbaee7003c8d45c2 2024-11-17T01:54:10,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/d60b336b277f406e8d182d1e333e928e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/d60b336b277f406e8d182d1e333e928e 2024-11-17T01:54:10,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/74a2445220ac4b93966f223d2123cb7b to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/74a2445220ac4b93966f223d2123cb7b 2024-11-17T01:54:10,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/980205ee49c14967b0741635219d7bf9 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/980205ee49c14967b0741635219d7bf9 2024-11-17T01:54:10,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/cabe72e632f142a682f3d60eb25e0556 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/cabe72e632f142a682f3d60eb25e0556 2024-11-17T01:54:10,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/d3e2c8cf94904ce58e30070151fa6e24 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/d3e2c8cf94904ce58e30070151fa6e24 2024-11-17T01:54:10,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/fdfd09a1f74a489fa9842cfe9a851c58 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/fdfd09a1f74a489fa9842cfe9a851c58 2024-11-17T01:54:10,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/ab4fed1b0ad74dae9954985d6d75d192 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/ab4fed1b0ad74dae9954985d6d75d192 2024-11-17T01:54:10,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/d32d8f6f34cd49e1bef482b17f1ddab3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/d32d8f6f34cd49e1bef482b17f1ddab3 2024-11-17T01:54:10,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/0c7cbbcb37af42adbea8f634f861f073 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/0c7cbbcb37af42adbea8f634f861f073 2024-11-17T01:54:10,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/71093efec090450c90979e9f14690403 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/71093efec090450c90979e9f14690403 2024-11-17T01:54:10,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/6ede9e50ec0b4fcf81a1570d18672fa2 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/6ede9e50ec0b4fcf81a1570d18672fa2 2024-11-17T01:54:10,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/0f49789e2b4c4ffbacf047b3646ee250 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/0f49789e2b4c4ffbacf047b3646ee250 2024-11-17T01:54:10,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/b76c2ac636e946078d065dc869baf92b to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/b76c2ac636e946078d065dc869baf92b 2024-11-17T01:54:10,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/4562e311f03f416b83b14d2a938b9f20 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/4562e311f03f416b83b14d2a938b9f20 2024-11-17T01:54:10,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/4195efd15eb44d0b8179efe60e2bea92 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/4195efd15eb44d0b8179efe60e2bea92 2024-11-17T01:54:10,729 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/recovered.edits/409.seqid, newMaxSeqId=409, maxSeqId=1 2024-11-17T01:54:10,730 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. 2024-11-17T01:54:10,730 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 5e67ace45ed3f9aacb3b2064d2a9351c: Waiting for close lock at 1731808448998Running coprocessor pre-close hooks at 1731808448998Disabling compacts and flushes for region at 1731808448998Disabling writes for close at 1731808449338 (+340 ms)Obtaining lock to block concurrent updates at 1731808449338Preparing flush snapshotting stores in 5e67ace45ed3f9aacb3b2064d2a9351c at 1731808449338Finished memstore snapshotting TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731808449339 (+1 ms)Flushing stores of TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c. at 1731808449339Flushing 5e67ace45ed3f9aacb3b2064d2a9351c/A: creating writer at 1731808449339Flushing 5e67ace45ed3f9aacb3b2064d2a9351c/A: appending metadata at 1731808449343 (+4 ms)Flushing 5e67ace45ed3f9aacb3b2064d2a9351c/A: closing flushed file at 1731808449343Flushing 5e67ace45ed3f9aacb3b2064d2a9351c/B: creating writer at 1731808449794 (+451 ms)Flushing 5e67ace45ed3f9aacb3b2064d2a9351c/B: appending metadata at 1731808449800 (+6 ms)Flushing 5e67ace45ed3f9aacb3b2064d2a9351c/B: closing flushed file at 1731808449800Flushing 5e67ace45ed3f9aacb3b2064d2a9351c/C: creating writer at 1731808450213 (+413 ms)Flushing 5e67ace45ed3f9aacb3b2064d2a9351c/C: appending metadata at 1731808450216 (+3 ms)Flushing 5e67ace45ed3f9aacb3b2064d2a9351c/C: closing flushed file at 1731808450217 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bccb2c4: reopening flushed file at 1731808450628 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fb3330f: reopening flushed file at 1731808450635 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4825f357: reopening flushed file at 1731808450642 (+7 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 5e67ace45ed3f9aacb3b2064d2a9351c in 1310ms, sequenceid=406, compaction requested=true at 1731808450648 (+6 ms)Writing region close event to WAL at 1731808450726 (+78 ms)Running coprocessor post-close hooks at 1731808450729 (+3 ms)Closed at 1731808450729 2024-11-17T01:54:10,732 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:54:10,733 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=5e67ace45ed3f9aacb3b2064d2a9351c, regionState=CLOSED 2024-11-17T01:54:10,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 because future has completed 2024-11-17T01:54:10,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-17T01:54:10,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 5e67ace45ed3f9aacb3b2064d2a9351c, server=c6c0d7a0a7c0,41157,1731808292173 in 1.8900 sec 2024-11-17T01:54:10,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-17T01:54:10,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e67ace45ed3f9aacb3b2064d2a9351c, UNASSIGN in 1.8970 sec 2024-11-17T01:54:10,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-17T01:54:10,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9010 sec 2024-11-17T01:54:10,742 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808450742"}]},"ts":"1731808450742"} 2024-11-17T01:54:10,743 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T01:54:10,743 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T01:54:10,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9110 sec 2024-11-17T01:54:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T01:54:10,985 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:54:10,986 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T01:54:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:54:10,990 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:54:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T01:54:10,991 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:54:10,995 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:54:10,999 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A, FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B, FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C, FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/recovered.edits] 2024-11-17T01:54:11,003 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/129519b83c074277bd8f10211ac0a5b4 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/129519b83c074277bd8f10211ac0a5b4 2024-11-17T01:54:11,004 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/48516f47ff0f4aa99ebb123263580932 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/48516f47ff0f4aa99ebb123263580932 2024-11-17T01:54:11,006 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/d0323f76f9f44f969bfd50fd70380bfb to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/A/d0323f76f9f44f969bfd50fd70380bfb 2024-11-17T01:54:11,009 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/06af3c4800df42f2b9dd132885476b76 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/06af3c4800df42f2b9dd132885476b76 2024-11-17T01:54:11,010 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/530bb48b00304b18b4588780dd1d1102 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/530bb48b00304b18b4588780dd1d1102 2024-11-17T01:54:11,012 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/fc76647868df496394997711c25cb740 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/B/fc76647868df496394997711c25cb740 2024-11-17T01:54:11,014 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/62f66185c21d435382730274a91179a0 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/62f66185c21d435382730274a91179a0 2024-11-17T01:54:11,015 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/a241b1f48f47456db6d4592f289b1dc2 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/a241b1f48f47456db6d4592f289b1dc2 2024-11-17T01:54:11,017 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/e000ab4207cb4379a975271672cc437b to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/C/e000ab4207cb4379a975271672cc437b 2024-11-17T01:54:11,020 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/recovered.edits/409.seqid to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c/recovered.edits/409.seqid 2024-11-17T01:54:11,020 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/5e67ace45ed3f9aacb3b2064d2a9351c 2024-11-17T01:54:11,021 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T01:54:11,023 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:54:11,026 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T01:54:11,029 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T01:54:11,030 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:54:11,030 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T01:54:11,030 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731808451030"}]},"ts":"9223372036854775807"} 2024-11-17T01:54:11,033 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T01:54:11,033 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5e67ace45ed3f9aacb3b2064d2a9351c, NAME => 'TestAcidGuarantees,,1731808425649.5e67ace45ed3f9aacb3b2064d2a9351c.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T01:54:11,033 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T01:54:11,033 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731808451033"}]},"ts":"9223372036854775807"} 2024-11-17T01:54:11,035 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T01:54:11,036 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:54:11,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-11-17T01:54:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T01:54:11,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T01:54:11,098 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:54:11,109 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251 (was 249) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=454 (was 464), ProcessCount=11 (was 11), AvailableMemoryMB=5467 (was 5827) 2024-11-17T01:54:11,118 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=454, ProcessCount=11, AvailableMemoryMB=5467 2024-11-17T01:54:11,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T01:54:11,120 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:54:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T01:54:11,121 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T01:54:11,121 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:11,122 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-17T01:54:11,122 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T01:54:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T01:54:11,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742352_1528 (size=1061) 2024-11-17T01:54:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T01:54:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T01:54:11,528 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1 2024-11-17T01:54:11,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742353_1529 (size=53) 2024-11-17T01:54:11,737 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T01:54:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T01:54:11,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:54:11,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f6dcaac235b7e538d527b651e63f9dcc, disabling compactions & flushes 2024-11-17T01:54:11,934 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:11,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:11,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. after waiting 0 ms 2024-11-17T01:54:11,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:11,934 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:11,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f6dcaac235b7e538d527b651e63f9dcc: Waiting for close lock at 1731808451934Disabling compacts and flushes for region at 1731808451934Disabling writes for close at 1731808451934Writing region close event to WAL at 1731808451934Closed at 1731808451934 2024-11-17T01:54:11,937 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T01:54:11,937 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731808451937"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731808451937"}]},"ts":"1731808451937"} 2024-11-17T01:54:11,940 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T01:54:11,941 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T01:54:11,941 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808451941"}]},"ts":"1731808451941"} 2024-11-17T01:54:11,943 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T01:54:11,944 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6dcaac235b7e538d527b651e63f9dcc, ASSIGN}] 2024-11-17T01:54:11,946 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6dcaac235b7e538d527b651e63f9dcc, ASSIGN 2024-11-17T01:54:11,947 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6dcaac235b7e538d527b651e63f9dcc, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,41157,1731808292173; forceNewPlan=false, retain=false 2024-11-17T01:54:12,098 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=f6dcaac235b7e538d527b651e63f9dcc, regionState=OPENING, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:12,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6dcaac235b7e538d527b651e63f9dcc, ASSIGN because future has completed 2024-11-17T01:54:12,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173}] 2024-11-17T01:54:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T01:54:12,260 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:12,260 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:54:12,261 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:12,261 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:54:12,261 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:12,261 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:12,264 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:12,267 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:54:12,267 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6dcaac235b7e538d527b651e63f9dcc columnFamilyName A 2024-11-17T01:54:12,267 DEBUG [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:12,268 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] regionserver.HStore(327): Store=f6dcaac235b7e538d527b651e63f9dcc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:54:12,269 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:12,270 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:54:12,271 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6dcaac235b7e538d527b651e63f9dcc columnFamilyName B 2024-11-17T01:54:12,271 DEBUG [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:12,272 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] regionserver.HStore(327): Store=f6dcaac235b7e538d527b651e63f9dcc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:54:12,272 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:12,274 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:54:12,274 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6dcaac235b7e538d527b651e63f9dcc columnFamilyName C 2024-11-17T01:54:12,274 DEBUG [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:12,274 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] regionserver.HStore(327): Store=f6dcaac235b7e538d527b651e63f9dcc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:54:12,274 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:12,274 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:12,275 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:12,275 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:12,277 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:12,277 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:12,277 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T01:54:12,278 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:12,280 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:54:12,280 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened f6dcaac235b7e538d527b651e63f9dcc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73061762, jitterRate=0.08870509266853333}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T01:54:12,280 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:12,281 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for f6dcaac235b7e538d527b651e63f9dcc: Running coprocessor pre-open hook at 1731808452261Writing region info on filesystem at 1731808452261Initializing all the Stores at 1731808452263 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808452263Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808452264 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808452264Cleaning up temporary data from old regions at 1731808452277 (+13 ms)Running coprocessor post-open hooks at 1731808452280 (+3 ms)Region opened successfully at 1731808452281 (+1 ms) 2024-11-17T01:54:12,282 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., pid=144, masterSystemTime=1731808452254 2024-11-17T01:54:12,284 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:12,284 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:12,284 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=f6dcaac235b7e538d527b651e63f9dcc, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:12,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 because future has completed 2024-11-17T01:54:12,287 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45023 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=c6c0d7a0a7c0,41157,1731808292173, table=TestAcidGuarantees, region=f6dcaac235b7e538d527b651e63f9dcc. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-17T01:54:12,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-17T01:54:12,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 in 186 msec 2024-11-17T01:54:12,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-17T01:54:12,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6dcaac235b7e538d527b651e63f9dcc, ASSIGN in 345 msec 2024-11-17T01:54:12,292 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T01:54:12,292 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808452292"}]},"ts":"1731808452292"} 2024-11-17T01:54:12,293 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T01:54:12,294 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T01:54:12,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1740 sec 2024-11-17T01:54:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T01:54:13,276 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:54:13,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a04de68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:13,277 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:54:13,278 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:54:13,279 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:54:13,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:54:13,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:54:13,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53afbea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:13,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:54:13,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:54:13,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:13,283 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:54:13,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1489a3de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:13,284 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:54:13,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:54:13,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:54:13,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:54:13,287 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T01:54:13,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is c6c0d7a0a7c0,45023,1731808291372 2024-11-17T01:54:13,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@65dfe3b8 2024-11-17T01:54:13,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T01:54:13,289 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T01:54:13,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T01:54:13,290 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:54:13,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T01:54:13,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742354_1530 (size=1097) 2024-11-17T01:54:13,705 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-17T01:54:13,705 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-17T01:54:13,707 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T01:54:13,709 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6dcaac235b7e538d527b651e63f9dcc, REOPEN/MOVE}] 2024-11-17T01:54:13,710 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6dcaac235b7e538d527b651e63f9dcc, REOPEN/MOVE 2024-11-17T01:54:13,710 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=f6dcaac235b7e538d527b651e63f9dcc, regionState=CLOSING, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:13,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6dcaac235b7e538d527b651e63f9dcc, REOPEN/MOVE because future has completed 2024-11-17T01:54:13,712 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T01:54:13,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173}] 2024-11-17T01:54:13,864 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:13,864 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T01:54:13,864 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing f6dcaac235b7e538d527b651e63f9dcc, disabling compactions & flushes 2024-11-17T01:54:13,864 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:13,864 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:13,864 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. after waiting 0 ms 2024-11-17T01:54:13,864 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:13,867 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T01:54:13,868 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:13,868 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for f6dcaac235b7e538d527b651e63f9dcc: Waiting for close lock at 1731808453864Running coprocessor pre-close hooks at 1731808453864Disabling compacts and flushes for region at 1731808453864Disabling writes for close at 1731808453864Writing region close event to WAL at 1731808453864Running coprocessor post-close hooks at 1731808453868 (+4 ms)Closed at 1731808453868 2024-11-17T01:54:13,868 WARN [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: f6dcaac235b7e538d527b651e63f9dcc to self. 2024-11-17T01:54:13,870 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:13,871 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=f6dcaac235b7e538d527b651e63f9dcc, regionState=CLOSED 2024-11-17T01:54:13,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 because future has completed 2024-11-17T01:54:13,876 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-17T01:54:13,876 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 in 161 msec 2024-11-17T01:54:13,877 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6dcaac235b7e538d527b651e63f9dcc, REOPEN/MOVE; state=CLOSED, location=c6c0d7a0a7c0,41157,1731808292173; forceNewPlan=false, retain=true 2024-11-17T01:54:14,027 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=f6dcaac235b7e538d527b651e63f9dcc, regionState=OPENING, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6dcaac235b7e538d527b651e63f9dcc, REOPEN/MOVE because future has completed 2024-11-17T01:54:14,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173}] 2024-11-17T01:54:14,189 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:14,189 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:54:14,190 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:14,190 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:54:14,190 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:14,190 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:14,193 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:14,194 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:54:14,195 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6dcaac235b7e538d527b651e63f9dcc columnFamilyName A 2024-11-17T01:54:14,196 DEBUG [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:14,196 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] regionserver.HStore(327): Store=f6dcaac235b7e538d527b651e63f9dcc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:54:14,196 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:14,197 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:54:14,197 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6dcaac235b7e538d527b651e63f9dcc columnFamilyName B 2024-11-17T01:54:14,197 DEBUG [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:14,197 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] regionserver.HStore(327): Store=f6dcaac235b7e538d527b651e63f9dcc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:54:14,197 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:14,198 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T01:54:14,198 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6dcaac235b7e538d527b651e63f9dcc columnFamilyName C 2024-11-17T01:54:14,198 DEBUG [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:14,198 INFO [StoreOpener-f6dcaac235b7e538d527b651e63f9dcc-1 {}] regionserver.HStore(327): Store=f6dcaac235b7e538d527b651e63f9dcc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:54:14,198 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:14,199 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:14,199 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:14,200 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:14,201 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:14,201 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:14,202 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T01:54:14,203 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:14,204 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened f6dcaac235b7e538d527b651e63f9dcc; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58736162, jitterRate=-0.12476298213005066}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T01:54:14,204 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:14,204 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for f6dcaac235b7e538d527b651e63f9dcc: Running coprocessor pre-open hook at 1731808454190Writing region info on filesystem at 1731808454190Initializing all the Stores at 1731808454192 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808454192Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808454193 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808454193Cleaning up temporary data from old regions at 1731808454201 (+8 ms)Running coprocessor post-open hooks at 1731808454204 (+3 ms)Region opened successfully at 1731808454204 2024-11-17T01:54:14,205 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., pid=149, masterSystemTime=1731808454183 2024-11-17T01:54:14,207 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:14,207 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:14,207 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=f6dcaac235b7e538d527b651e63f9dcc, regionState=OPEN, openSeqNum=5, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 because future has completed 2024-11-17T01:54:14,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-17T01:54:14,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 in 179 msec 2024-11-17T01:54:14,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-17T01:54:14,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6dcaac235b7e538d527b651e63f9dcc, REOPEN/MOVE in 502 msec 2024-11-17T01:54:14,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-17T01:54:14,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 507 msec 2024-11-17T01:54:14,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 927 msec 2024-11-17T01:54:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-17T01:54:14,220 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:54:14,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:54:14,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:54:14,221 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:54:14,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:54:14,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:54:14,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@569ad71d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:54:14,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:54:14,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:14,222 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:54:14,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55931284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,223 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:54:14,224 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:54:14,224 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:54:14,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:54:14,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:54:14,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b1155aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:54:14,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:54:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:14,225 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:54:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d04d0e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,226 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:54:14,226 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:54:14,227 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:54:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:54:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:54:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b75919f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:54:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:54:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:14,228 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:54:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6343769a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,229 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:54:14,229 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:54:14,230 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:54:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:54:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:54:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cb6d26e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:54:14,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:54:14,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:14,231 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:54:14,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7874638a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,233 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:54:14,233 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:54:14,234 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:54:14,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:54:14,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:54:14,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510937da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:54:14,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:54:14,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:14,236 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:54:14,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6520b195, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,237 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:54:14,237 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:54:14,238 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:54:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:54:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:54:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a826a50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:54:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:54:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:14,240 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:54:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a5d40db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,241 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:54:14,241 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:54:14,242 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:54:14,242 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:54:14,242 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:54:14,242 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@324e40fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,242 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:54:14,242 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:54:14,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:14,243 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:54:14,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3720307b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,245 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:54:14,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:54:14,247 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:54:14,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:54:14,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:54:14,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c021c46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:54:14,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:54:14,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:14,249 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:54:14,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a567d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,250 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,250 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:54:14,250 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:54:14,251 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:54:14,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:54:14,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:54:14,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25f694a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:54:14,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:54:14,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:14,252 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:54:14,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22bc3576, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,253 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,45023,-1 for getting cluster id 2024-11-17T01:54:14,253 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:54:14,253 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48ead001-edd0-44c1-827a-ff10abe48617' 2024-11-17T01:54:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:54:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48ead001-edd0-44c1-827a-ff10abe48617" 2024-11-17T01:54:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f32c70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,45023,-1] 2024-11-17T01:54:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:54:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:14,254 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:54:14,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aeb1775, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:54:14,255 DEBUG [Thread-2381 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:54:14,255 DEBUG [Thread-2385 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:54:14,256 DEBUG [Thread-2379 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:54:14,256 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:54:14,256 DEBUG [Thread-2370 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:54:14,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:54:14,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:54:14,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:54:14,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:54:14,256 DEBUG [Thread-2374 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:54:14,256 DEBUG [Thread-2377 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:54:14,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:54:14,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:54:14,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:54:14,257 DEBUG [Thread-2366 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:54:14,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:54:14,257 DEBUG [Thread-2383 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:54:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:54:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:54:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:54:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:54:14,257 DEBUG [Thread-2372 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:54:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:54:14,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:54:14,258 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:54:14,258 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:54:14,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:54:14,258 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:54:14,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:54:14,258 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:54:14,258 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:54:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-17T01:54:14,259 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:54:14,259 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:54:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T01:54:14,259 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:54:14,259 DEBUG [Thread-2368 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:54:14,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:54:14,260 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:54:14,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:54:14,260 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:54:14,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:54:14,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:54:14,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:54:14,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:54:14,261 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:54:14,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:54:14,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:54:14,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=-1] 2024-11-17T01:54:14,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:54:14,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:54:14,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:54:14,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:54:14,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:54:14,264 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:54:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:14,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6dcaac235b7e538d527b651e63f9dcc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T01:54:14,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=A 2024-11-17T01:54:14,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:14,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=B 2024-11-17T01:54:14,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:14,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=C 2024-11-17T01:54:14,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:14,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5] 2024-11-17T01:54:14,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808454277, max timestamp=1731808454286Num uniques 859; 2024-11-17T01:54:14,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808454277, max timestamp=1731808454286Num uniques 859; 2024-11-17T01:54:14,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:14,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731808454277, max timestamp=1731808454286Num uniques 859; 2024-11-17T01:54:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808514285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:54:14,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:54:14,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T01:54:14,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:14,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173a64620444b64e05bb19a9cbe45a471d_f6dcaac235b7e538d527b651e63f9dcc is 50, key is test_row_0/A:col10/1731808454267/Put/seqid=0 2024-11-17T01:54:14,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742355_1531 (size=14594) 2024-11-17T01:54:14,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:14,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57256 deadline: 1731808514333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57268 deadline: 1731808514332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:14,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808514337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:14,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808514337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:14,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T01:54:14,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808514394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:14,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T01:54:14,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:14,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:14,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:14,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:14,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:14,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57268 deadline: 1731808514444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57256 deadline: 1731808514444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808514444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808514444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:14,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T01:54:14,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:14,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:14,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:14,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:14,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T01:54:14,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808514604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:14,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57268 deadline: 1731808514654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57256 deadline: 1731808514654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:14,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808514654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808514654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:14,695 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:14,697 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173a64620444b64e05bb19a9cbe45a471d_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173a64620444b64e05bb19a9cbe45a471d_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:14,698 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/35c017629e1849ffadc07f3091a620e1, store: [table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:14,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/35c017629e1849ffadc07f3091a620e1 is 175, key is test_row_0/A:col10/1731808454267/Put/seqid=0 2024-11-17T01:54:14,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742356_1532 (size=39549) 2024-11-17T01:54:14,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T01:54:14,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:14,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:14,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:14,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:14,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:14,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T01:54:14,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:14,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:14,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:14,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:14,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T01:54:14,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808514914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:14,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57268 deadline: 1731808514964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57256 deadline: 1731808514964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808514964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808514964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:15,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T01:54:15,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:15,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:15,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:15,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:15,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:15,102 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/35c017629e1849ffadc07f3091a620e1 2024-11-17T01:54:15,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/2c11150aba694de1a8ca5906db6b4465 is 50, key is test_row_0/B:col10/1731808454267/Put/seqid=0 2024-11-17T01:54:15,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742357_1533 (size=12001) 2024-11-17T01:54:15,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T01:54:15,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:15,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:15,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:15,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:15,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:15,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T01:54:15,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:15,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:15,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:15,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:15,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:15,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T01:54:15,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808515423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:15,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57268 deadline: 1731808515474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:15,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:15,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:15,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:15,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57256 deadline: 1731808515474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:15,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808515474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:15,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808515474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:15,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/2c11150aba694de1a8ca5906db6b4465 2024-11-17T01:54:15,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T01:54:15,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:15,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:15,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:15,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:15,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/89c4639b336a494295a4ad3251bc20e5 is 50, key is test_row_0/C:col10/1731808454267/Put/seqid=0 2024-11-17T01:54:15,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742358_1534 (size=12001) 2024-11-17T01:54:15,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T01:54:15,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:15,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:15,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:15,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:15,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:15,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T01:54:15,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:15,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:15,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:15,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:15,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:15,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:15,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/89c4639b336a494295a4ad3251bc20e5 2024-11-17T01:54:15,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/35c017629e1849ffadc07f3091a620e1 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/35c017629e1849ffadc07f3091a620e1 2024-11-17T01:54:15,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/35c017629e1849ffadc07f3091a620e1, entries=200, sequenceid=14, filesize=38.6 K 2024-11-17T01:54:15,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/2c11150aba694de1a8ca5906db6b4465 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/2c11150aba694de1a8ca5906db6b4465 2024-11-17T01:54:15,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/2c11150aba694de1a8ca5906db6b4465, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T01:54:15,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/89c4639b336a494295a4ad3251bc20e5 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/89c4639b336a494295a4ad3251bc20e5 2024-11-17T01:54:15,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/89c4639b336a494295a4ad3251bc20e5, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T01:54:15,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f6dcaac235b7e538d527b651e63f9dcc in 1697ms, sequenceid=14, compaction requested=false 2024-11-17T01:54:15,965 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T01:54:15,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:15,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T01:54:15,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:15,990 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing f6dcaac235b7e538d527b651e63f9dcc 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-17T01:54:15,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=A 2024-11-17T01:54:15,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:54:15,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=B 2024-11-17T01:54:15,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:54:15,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=C 2024-11-17T01:54:15,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:54:16,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172e9d9d165cbc49c289a7d8694890e139_f6dcaac235b7e538d527b651e63f9dcc is 50, key is test_row_0/A:col10/1731808454335/Put/seqid=0 2024-11-17T01:54:16,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742359_1535 (size=12154) 2024-11-17T01:54:16,245 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T01:54:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T01:54:16,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:16,413 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172e9d9d165cbc49c289a7d8694890e139_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172e9d9d165cbc49c289a7d8694890e139_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:16,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/86d1cee2d8014cf3a21427f64320cc47, store: [table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:16,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/86d1cee2d8014cf3a21427f64320cc47 is 175, key is test_row_0/A:col10/1731808454335/Put/seqid=0 2024-11-17T01:54:16,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742360_1536 (size=30955) 2024-11-17T01:54:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:16,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:16,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808516459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:16,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:16,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:16,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:16,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57268 deadline: 1731808516484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:16,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808516484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:16,486 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in f6dcaac235b7e538d527b651e63f9dcc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:16,486 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in f6dcaac235b7e538d527b651e63f9dcc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:16,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57256 deadline: 1731808516494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:16,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:16,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808516494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:16,498 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_2' in f6dcaac235b7e538d527b651e63f9dcc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:16,498 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in f6dcaac235b7e538d527b651e63f9dcc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:16,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:16,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:16,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:16,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:16,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:16,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:16,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808516574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:16,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:16,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808516784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:16,818 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/86d1cee2d8014cf3a21427f64320cc47 2024-11-17T01:54:16,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/0264caa5d94542299980416d1da0600e is 50, key is test_row_0/B:col10/1731808454335/Put/seqid=0 2024-11-17T01:54:16,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742361_1537 (size=12001) 2024-11-17T01:54:17,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808517094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:17,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:17,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:17,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:17,230 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/0264caa5d94542299980416d1da0600e 2024-11-17T01:54:17,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/05aac7bd4acf41f3be9bdb4970db148e is 50, key is test_row_0/C:col10/1731808454335/Put/seqid=0 2024-11-17T01:54:17,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742362_1538 (size=12001) 2024-11-17T01:54:17,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:17,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808517604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:17,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:17,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:17,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:17,642 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/05aac7bd4acf41f3be9bdb4970db148e 2024-11-17T01:54:17,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/86d1cee2d8014cf3a21427f64320cc47 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/86d1cee2d8014cf3a21427f64320cc47 2024-11-17T01:54:17,648 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/86d1cee2d8014cf3a21427f64320cc47, entries=150, sequenceid=50, filesize=30.2 K 2024-11-17T01:54:17,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/0264caa5d94542299980416d1da0600e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/0264caa5d94542299980416d1da0600e 2024-11-17T01:54:17,651 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/0264caa5d94542299980416d1da0600e, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T01:54:17,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/05aac7bd4acf41f3be9bdb4970db148e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/05aac7bd4acf41f3be9bdb4970db148e 2024-11-17T01:54:17,654 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/05aac7bd4acf41f3be9bdb4970db148e, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T01:54:17,654 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for f6dcaac235b7e538d527b651e63f9dcc in 1664ms, sequenceid=50, compaction requested=false 2024-11-17T01:54:17,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:17,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:17,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-17T01:54:17,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-17T01:54:17,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-17T01:54:17,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3960 sec 2024-11-17T01:54:17,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 3.4010 sec 2024-11-17T01:54:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T01:54:18,414 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:54:18,414 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:54:18,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-17T01:54:18,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T01:54:18,416 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:54:18,416 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:54:18,416 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:54:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:18,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6dcaac235b7e538d527b651e63f9dcc 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T01:54:18,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=A 2024-11-17T01:54:18,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:18,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=B 2024-11-17T01:54:18,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:18,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=C 2024-11-17T01:54:18,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:18,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178e1a1ecdc8d54b57b1ab6986b23c7496_f6dcaac235b7e538d527b651e63f9dcc is 50, key is test_row_0/A:col10/1731808458495/Put/seqid=0 2024-11-17T01:54:18,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742363_1539 (size=14594) 2024-11-17T01:54:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T01:54:18,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808518527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:18,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731808458519, max timestamp=1731808458530Num uniques 859; 2024-11-17T01:54:18,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T01:54:18,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731808458519, max timestamp=1731808458530Num uniques 859; 2024-11-17T01:54:18,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731808458519, max timestamp=1731808458530Num uniques 859; 2024-11-17T01:54:18,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T01:54:18,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/c6c0d7a0a7c0:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T01:54:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:18,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57256 deadline: 1731808518554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:18,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808518555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:18,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,560 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:18,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:18,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57268 deadline: 1731808518559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:18,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T01:54:18,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:18,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:18,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:18,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:18,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:18,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808518614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:18,617 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in f6dcaac235b7e538d527b651e63f9dcc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:18,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808518644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:18,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57256 deadline: 1731808518664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:18,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808518664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:18,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57268 deadline: 1731808518674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:18,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:18,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T01:54:18,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:18,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:18,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:18,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:18,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T01:54:18,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808518854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:18,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:18,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T01:54:18,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:18,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:18,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:18,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:18,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:18,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808518874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:18,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:18,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57256 deadline: 1731808518874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:18,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:18,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57268 deadline: 1731808518884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:18,905 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:18,908 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178e1a1ecdc8d54b57b1ab6986b23c7496_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178e1a1ecdc8d54b57b1ab6986b23c7496_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:18,909 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/1866a8f327104812ba09e3590103d1d2, store: [table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:18,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/1866a8f327104812ba09e3590103d1d2 is 175, key is test_row_0/A:col10/1731808458495/Put/seqid=0 2024-11-17T01:54:18,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742364_1540 (size=39549) 2024-11-17T01:54:19,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T01:54:19,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:19,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:19,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:19,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:19,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T01:54:19,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808519164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:19,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:19,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:19,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:19,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T01:54:19,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:19,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:19,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:19,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:19,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:19,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808519184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:19,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57256 deadline: 1731808519184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:19,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57268 deadline: 1731808519194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:19,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:19,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:19,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:19,312 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/1866a8f327104812ba09e3590103d1d2 2024-11-17T01:54:19,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/e89e6a6ffbbc4e2f8237dd6525c52369 is 50, key is test_row_0/B:col10/1731808458495/Put/seqid=0 2024-11-17T01:54:19,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742365_1541 (size=12001) 2024-11-17T01:54:19,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T01:54:19,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:19,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:19,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:19,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:19,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:19,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T01:54:19,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:19,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:19,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:19,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:19,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:19,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T01:54:19,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T01:54:19,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:19,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:19,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:19,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:19,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:19,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:19,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:19,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808519674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:19,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:19,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:19,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:19,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808519694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:19,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:19,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:19,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:19,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57256 deadline: 1731808519694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:19,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:19,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:19,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:19,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:19,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57268 deadline: 1731808519704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:19,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/e89e6a6ffbbc4e2f8237dd6525c52369 2024-11-17T01:54:19,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/21d2360b268643c0a3e724fc9d37422a is 50, key is test_row_0/C:col10/1731808458495/Put/seqid=0 2024-11-17T01:54:19,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742366_1542 (size=12001) 2024-11-17T01:54:19,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/21d2360b268643c0a3e724fc9d37422a 2024-11-17T01:54:19,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/1866a8f327104812ba09e3590103d1d2 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/1866a8f327104812ba09e3590103d1d2 2024-11-17T01:54:19,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/1866a8f327104812ba09e3590103d1d2, entries=200, sequenceid=61, filesize=38.6 K 2024-11-17T01:54:19,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/e89e6a6ffbbc4e2f8237dd6525c52369 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/e89e6a6ffbbc4e2f8237dd6525c52369 2024-11-17T01:54:19,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/e89e6a6ffbbc4e2f8237dd6525c52369, entries=150, sequenceid=61, filesize=11.7 K 2024-11-17T01:54:19,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/21d2360b268643c0a3e724fc9d37422a as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/21d2360b268643c0a3e724fc9d37422a 2024-11-17T01:54:19,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/21d2360b268643c0a3e724fc9d37422a, entries=150, sequenceid=61, filesize=11.7 K 2024-11-17T01:54:19,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for f6dcaac235b7e538d527b651e63f9dcc in 1248ms, sequenceid=61, compaction requested=true 2024-11-17T01:54:19,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:19,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6dcaac235b7e538d527b651e63f9dcc:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:54:19,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:19,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6dcaac235b7e538d527b651e63f9dcc:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:54:19,744 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:19,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:19,744 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:19,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6dcaac235b7e538d527b651e63f9dcc:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:54:19,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:54:19,745 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:19,745 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:19,745 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): f6dcaac235b7e538d527b651e63f9dcc/B is initiating minor compaction (all files) 2024-11-17T01:54:19,745 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): f6dcaac235b7e538d527b651e63f9dcc/A is initiating minor compaction (all files) 2024-11-17T01:54:19,745 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6dcaac235b7e538d527b651e63f9dcc/B in TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:19,745 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6dcaac235b7e538d527b651e63f9dcc/A in TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:19,745 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/2c11150aba694de1a8ca5906db6b4465, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/0264caa5d94542299980416d1da0600e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/e89e6a6ffbbc4e2f8237dd6525c52369] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp, totalSize=35.2 K 2024-11-17T01:54:19,745 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/35c017629e1849ffadc07f3091a620e1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/86d1cee2d8014cf3a21427f64320cc47, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/1866a8f327104812ba09e3590103d1d2] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp, totalSize=107.5 K 2024-11-17T01:54:19,745 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:19,745 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/35c017629e1849ffadc07f3091a620e1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/86d1cee2d8014cf3a21427f64320cc47, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/1866a8f327104812ba09e3590103d1d2] 2024-11-17T01:54:19,746 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c11150aba694de1a8ca5906db6b4465, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731808454264 2024-11-17T01:54:19,746 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35c017629e1849ffadc07f3091a620e1, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731808454264 2024-11-17T01:54:19,746 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 0264caa5d94542299980416d1da0600e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731808454329 2024-11-17T01:54:19,746 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86d1cee2d8014cf3a21427f64320cc47, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731808454329 2024-11-17T01:54:19,747 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting e89e6a6ffbbc4e2f8237dd6525c52369, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731808456447 2024-11-17T01:54:19,747 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1866a8f327104812ba09e3590103d1d2, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731808456447 2024-11-17T01:54:19,752 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6dcaac235b7e538d527b651e63f9dcc#B#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:19,753 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/9368126a778645b1adaa15cd37ba0e18 is 50, key is test_row_0/B:col10/1731808458495/Put/seqid=0 2024-11-17T01:54:19,754 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:19,759 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111731c7deda521d4e578156f5ebaed151aa_f6dcaac235b7e538d527b651e63f9dcc store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:19,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742367_1543 (size=12104) 2024-11-17T01:54:19,761 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111731c7deda521d4e578156f5ebaed151aa_f6dcaac235b7e538d527b651e63f9dcc, store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:19,761 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111731c7deda521d4e578156f5ebaed151aa_f6dcaac235b7e538d527b651e63f9dcc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:19,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742368_1544 (size=4469) 2024-11-17T01:54:19,764 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6dcaac235b7e538d527b651e63f9dcc#A#compaction#461 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:19,765 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/df46b6d2b8e041b0b5e3f18c1d729b4a is 175, key is test_row_0/A:col10/1731808458495/Put/seqid=0 2024-11-17T01:54:19,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742369_1545 (size=31058) 2024-11-17T01:54:19,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T01:54:19,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:19,789 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing f6dcaac235b7e538d527b651e63f9dcc 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-17T01:54:19,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=A 2024-11-17T01:54:19,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:54:19,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=B 2024-11-17T01:54:19,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:54:19,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=C 2024-11-17T01:54:19,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T01:54:19,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f48fa1b739dd4adfa4dbacf03ff0c6bc_f6dcaac235b7e538d527b651e63f9dcc is 50, key is test_row_0/A:col10/1731808458558/Put/seqid=0 2024-11-17T01:54:19,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742370_1546 (size=12154) 2024-11-17T01:54:20,163 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/9368126a778645b1adaa15cd37ba0e18 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/9368126a778645b1adaa15cd37ba0e18 2024-11-17T01:54:20,166 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6dcaac235b7e538d527b651e63f9dcc/B of f6dcaac235b7e538d527b651e63f9dcc into 9368126a778645b1adaa15cd37ba0e18(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:20,166 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:20,166 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., storeName=f6dcaac235b7e538d527b651e63f9dcc/B, priority=13, startTime=1731808459744; duration=0sec 2024-11-17T01:54:20,166 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:54:20,166 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6dcaac235b7e538d527b651e63f9dcc:B 2024-11-17T01:54:20,167 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:20,167 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:20,167 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): f6dcaac235b7e538d527b651e63f9dcc/C is initiating minor compaction (all files) 2024-11-17T01:54:20,167 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6dcaac235b7e538d527b651e63f9dcc/C in TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:20,167 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/89c4639b336a494295a4ad3251bc20e5, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/05aac7bd4acf41f3be9bdb4970db148e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/21d2360b268643c0a3e724fc9d37422a] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp, totalSize=35.2 K 2024-11-17T01:54:20,168 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 89c4639b336a494295a4ad3251bc20e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731808454264 2024-11-17T01:54:20,168 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 05aac7bd4acf41f3be9bdb4970db148e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731808454329 2024-11-17T01:54:20,168 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 21d2360b268643c0a3e724fc9d37422a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731808456447 2024-11-17T01:54:20,171 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/df46b6d2b8e041b0b5e3f18c1d729b4a as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/df46b6d2b8e041b0b5e3f18c1d729b4a 2024-11-17T01:54:20,174 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6dcaac235b7e538d527b651e63f9dcc/A of f6dcaac235b7e538d527b651e63f9dcc into df46b6d2b8e041b0b5e3f18c1d729b4a(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:20,174 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:20,174 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., storeName=f6dcaac235b7e538d527b651e63f9dcc/A, priority=13, startTime=1731808459744; duration=0sec 2024-11-17T01:54:20,174 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6dcaac235b7e538d527b651e63f9dcc#C#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:20,174 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:20,174 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6dcaac235b7e538d527b651e63f9dcc:A 2024-11-17T01:54:20,175 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/572312267a704bdcb2852597a475fea3 is 50, key is test_row_0/C:col10/1731808458495/Put/seqid=0 2024-11-17T01:54:20,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742371_1547 (size=12104) 2024-11-17T01:54:20,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:20,200 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f48fa1b739dd4adfa4dbacf03ff0c6bc_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f48fa1b739dd4adfa4dbacf03ff0c6bc_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:20,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/342fce7fb08e4e378d757bfc72da8838, store: [table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:20,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/342fce7fb08e4e378d757bfc72da8838 is 175, key is test_row_0/A:col10/1731808458558/Put/seqid=0 2024-11-17T01:54:20,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742372_1548 (size=30955) 2024-11-17T01:54:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T01:54:20,581 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/572312267a704bdcb2852597a475fea3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/572312267a704bdcb2852597a475fea3 2024-11-17T01:54:20,584 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6dcaac235b7e538d527b651e63f9dcc/C of f6dcaac235b7e538d527b651e63f9dcc into 572312267a704bdcb2852597a475fea3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:20,584 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:20,584 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., storeName=f6dcaac235b7e538d527b651e63f9dcc/C, priority=13, startTime=1731808459744; duration=0sec 2024-11-17T01:54:20,584 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:20,584 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6dcaac235b7e538d527b651e63f9dcc:C 2024-11-17T01:54:20,605 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/342fce7fb08e4e378d757bfc72da8838 2024-11-17T01:54:20,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/45f5c60a367a472ea0db53d99155343d is 50, key is test_row_0/B:col10/1731808458558/Put/seqid=0 2024-11-17T01:54:20,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742373_1549 (size=12001) 2024-11-17T01:54:20,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:20,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:20,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808520666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:20,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:20,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:20,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808520684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:20,687 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in f6dcaac235b7e538d527b651e63f9dcc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:20,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808520703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:20,707 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in f6dcaac235b7e538d527b651e63f9dcc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:20,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57256 deadline: 1731808520714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:20,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57268 deadline: 1731808520714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:20,719 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in f6dcaac235b7e538d527b651e63f9dcc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:20,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:20,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:20,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:20,719 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in f6dcaac235b7e538d527b651e63f9dcc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:20,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:20,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:20,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:20,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808520774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:20,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:20,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808520984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:20,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:20,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:20,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:21,015 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/45f5c60a367a472ea0db53d99155343d 2024-11-17T01:54:21,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/a512c04aae9e4ef39e8386e0e3b23186 is 50, key is test_row_0/C:col10/1731808458558/Put/seqid=0 2024-11-17T01:54:21,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742374_1550 (size=12001) 2024-11-17T01:54:21,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:21,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808521294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:21,423 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/a512c04aae9e4ef39e8386e0e3b23186 2024-11-17T01:54:21,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/342fce7fb08e4e378d757bfc72da8838 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/342fce7fb08e4e378d757bfc72da8838 2024-11-17T01:54:21,429 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/342fce7fb08e4e378d757bfc72da8838, entries=150, sequenceid=95, filesize=30.2 K 2024-11-17T01:54:21,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/45f5c60a367a472ea0db53d99155343d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/45f5c60a367a472ea0db53d99155343d 2024-11-17T01:54:21,432 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/45f5c60a367a472ea0db53d99155343d, entries=150, sequenceid=95, filesize=11.7 K 2024-11-17T01:54:21,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/a512c04aae9e4ef39e8386e0e3b23186 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/a512c04aae9e4ef39e8386e0e3b23186 2024-11-17T01:54:21,435 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/a512c04aae9e4ef39e8386e0e3b23186, entries=150, sequenceid=95, filesize=11.7 K 2024-11-17T01:54:21,436 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for f6dcaac235b7e538d527b651e63f9dcc in 1647ms, sequenceid=95, compaction requested=false 2024-11-17T01:54:21,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:21,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:21,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-17T01:54:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-17T01:54:21,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-17T01:54:21,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0210 sec 2024-11-17T01:54:21,441 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 3.0260 sec 2024-11-17T01:54:21,737 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T01:54:21,737 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-17T01:54:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:21,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6dcaac235b7e538d527b651e63f9dcc 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T01:54:21,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=A 2024-11-17T01:54:21,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=B 2024-11-17T01:54:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=C 2024-11-17T01:54:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:21,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111780113dfb03f94a80b31787fcbc53cb16_f6dcaac235b7e538d527b651e63f9dcc is 50, key is test_row_0/A:col10/1731808460665/Put/seqid=0 2024-11-17T01:54:21,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742375_1551 (size=14594) 2024-11-17T01:54:21,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808521876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:21,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:21,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:21,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:21,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808521984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:21,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:21,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:21,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:22,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808522194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:22,225 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:22,228 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111780113dfb03f94a80b31787fcbc53cb16_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111780113dfb03f94a80b31787fcbc53cb16_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:22,228 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/e7090818afc24abd940be41c1dc35c2d, store: [table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:22,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/e7090818afc24abd940be41c1dc35c2d is 175, key is test_row_0/A:col10/1731808460665/Put/seqid=0 2024-11-17T01:54:22,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742376_1552 (size=39549) 2024-11-17T01:54:22,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808522504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:22,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:22,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:22,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T01:54:22,585 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:54:22,585 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:54:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-17T01:54:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T01:54:22,587 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:54:22,587 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:54:22,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:54:22,632 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/e7090818afc24abd940be41c1dc35c2d 2024-11-17T01:54:22,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/9ad15290ede44f8782d62aa129700f34 is 50, key is test_row_0/B:col10/1731808460665/Put/seqid=0 2024-11-17T01:54:22,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742377_1553 (size=12001) 2024-11-17T01:54:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T01:54:22,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808522704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:22,705 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in f6dcaac235b7e538d527b651e63f9dcc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:22,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:22,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:22,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:22,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808522724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:22,726 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in f6dcaac235b7e538d527b651e63f9dcc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:22,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57268 deadline: 1731808522734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:22,737 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in f6dcaac235b7e538d527b651e63f9dcc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:22,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T01:54:22,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:22,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:22,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:22,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:22,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:22,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57256 deadline: 1731808522744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:22,746 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in f6dcaac235b7e538d527b651e63f9dcc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:22,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:22,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:22,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:22,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T01:54:22,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:22,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:22,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:22,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:22,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T01:54:23,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808523013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:23,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/9ad15290ede44f8782d62aa129700f34 2024-11-17T01:54:23,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T01:54:23,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:23,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:23,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:23,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:23,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:23,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/b4cbb3f998d44d528e07f89b96cc7fdf is 50, key is test_row_0/C:col10/1731808460665/Put/seqid=0 2024-11-17T01:54:23,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742378_1554 (size=12001) 2024-11-17T01:54:23,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T01:54:23,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:23,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:23,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:23,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:23,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T01:54:23,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T01:54:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:23,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:23,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/b4cbb3f998d44d528e07f89b96cc7fdf 2024-11-17T01:54:23,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/e7090818afc24abd940be41c1dc35c2d as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/e7090818afc24abd940be41c1dc35c2d 2024-11-17T01:54:23,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/e7090818afc24abd940be41c1dc35c2d, entries=200, sequenceid=110, filesize=38.6 K 2024-11-17T01:54:23,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/9ad15290ede44f8782d62aa129700f34 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/9ad15290ede44f8782d62aa129700f34 2024-11-17T01:54:23,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/9ad15290ede44f8782d62aa129700f34, entries=150, sequenceid=110, filesize=11.7 K 2024-11-17T01:54:23,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/b4cbb3f998d44d528e07f89b96cc7fdf as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/b4cbb3f998d44d528e07f89b96cc7fdf 2024-11-17T01:54:23,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/b4cbb3f998d44d528e07f89b96cc7fdf, entries=150, sequenceid=110, filesize=11.7 K 2024-11-17T01:54:23,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f6dcaac235b7e538d527b651e63f9dcc in 1656ms, sequenceid=110, compaction requested=true 2024-11-17T01:54:23,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:23,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6dcaac235b7e538d527b651e63f9dcc:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:54:23,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:23,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6dcaac235b7e538d527b651e63f9dcc:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:54:23,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:23,464 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:23,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6dcaac235b7e538d527b651e63f9dcc:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:54:23,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:54:23,464 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:23,465 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:23,465 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:23,465 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): f6dcaac235b7e538d527b651e63f9dcc/A is initiating minor compaction (all files) 2024-11-17T01:54:23,465 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6dcaac235b7e538d527b651e63f9dcc/A in TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:23,465 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/df46b6d2b8e041b0b5e3f18c1d729b4a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/342fce7fb08e4e378d757bfc72da8838, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/e7090818afc24abd940be41c1dc35c2d] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp, totalSize=99.2 K 2024-11-17T01:54:23,465 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:23,465 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/df46b6d2b8e041b0b5e3f18c1d729b4a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/342fce7fb08e4e378d757bfc72da8838, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/e7090818afc24abd940be41c1dc35c2d] 2024-11-17T01:54:23,465 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): f6dcaac235b7e538d527b651e63f9dcc/B is initiating minor compaction (all files) 2024-11-17T01:54:23,465 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6dcaac235b7e538d527b651e63f9dcc/B in TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:23,465 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/9368126a778645b1adaa15cd37ba0e18, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/45f5c60a367a472ea0db53d99155343d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/9ad15290ede44f8782d62aa129700f34] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp, totalSize=35.3 K 2024-11-17T01:54:23,465 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting df46b6d2b8e041b0b5e3f18c1d729b4a, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731808456447 2024-11-17T01:54:23,465 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 9368126a778645b1adaa15cd37ba0e18, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731808456447 2024-11-17T01:54:23,466 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 342fce7fb08e4e378d757bfc72da8838, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731808458546 2024-11-17T01:54:23,466 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 45f5c60a367a472ea0db53d99155343d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731808458546 2024-11-17T01:54:23,466 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7090818afc24abd940be41c1dc35c2d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731808460652 2024-11-17T01:54:23,466 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ad15290ede44f8782d62aa129700f34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731808460652 2024-11-17T01:54:23,470 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:23,472 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6dcaac235b7e538d527b651e63f9dcc#B#compaction#470 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:23,472 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/5d836060fd9e46cba1ef9fc7558592ed is 50, key is test_row_0/B:col10/1731808460665/Put/seqid=0 2024-11-17T01:54:23,477 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117ad85e6dd574d4d5b83871c8318cf0b72_f6dcaac235b7e538d527b651e63f9dcc store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:23,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742379_1555 (size=12207) 2024-11-17T01:54:23,478 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117ad85e6dd574d4d5b83871c8318cf0b72_f6dcaac235b7e538d527b651e63f9dcc, store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:23,478 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ad85e6dd574d4d5b83871c8318cf0b72_f6dcaac235b7e538d527b651e63f9dcc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:23,481 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/5d836060fd9e46cba1ef9fc7558592ed as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/5d836060fd9e46cba1ef9fc7558592ed 2024-11-17T01:54:23,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742380_1556 (size=4469) 2024-11-17T01:54:23,483 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6dcaac235b7e538d527b651e63f9dcc#A#compaction#469 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:23,483 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/c9812ccbc00a4f84b0016804e29a77fa is 175, key is test_row_0/A:col10/1731808460665/Put/seqid=0 2024-11-17T01:54:23,485 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6dcaac235b7e538d527b651e63f9dcc/B of f6dcaac235b7e538d527b651e63f9dcc into 5d836060fd9e46cba1ef9fc7558592ed(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:23,485 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:23,485 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., storeName=f6dcaac235b7e538d527b651e63f9dcc/B, priority=13, startTime=1731808463464; duration=0sec 2024-11-17T01:54:23,485 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:54:23,485 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6dcaac235b7e538d527b651e63f9dcc:B 2024-11-17T01:54:23,485 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:23,486 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:23,486 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): f6dcaac235b7e538d527b651e63f9dcc/C is initiating minor compaction (all files) 2024-11-17T01:54:23,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742381_1557 (size=31161) 2024-11-17T01:54:23,486 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6dcaac235b7e538d527b651e63f9dcc/C in TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:23,486 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/572312267a704bdcb2852597a475fea3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/a512c04aae9e4ef39e8386e0e3b23186, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/b4cbb3f998d44d528e07f89b96cc7fdf] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp, totalSize=35.3 K 2024-11-17T01:54:23,487 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 572312267a704bdcb2852597a475fea3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731808456447 2024-11-17T01:54:23,487 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting a512c04aae9e4ef39e8386e0e3b23186, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731808458546 2024-11-17T01:54:23,488 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting b4cbb3f998d44d528e07f89b96cc7fdf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731808460652 2024-11-17T01:54:23,490 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/c9812ccbc00a4f84b0016804e29a77fa as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/c9812ccbc00a4f84b0016804e29a77fa 2024-11-17T01:54:23,493 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6dcaac235b7e538d527b651e63f9dcc#C#compaction#471 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:23,493 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/12e4824ce531494ab9b93bc2fba442f8 is 50, key is test_row_0/C:col10/1731808460665/Put/seqid=0 2024-11-17T01:54:23,496 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6dcaac235b7e538d527b651e63f9dcc/A of f6dcaac235b7e538d527b651e63f9dcc into c9812ccbc00a4f84b0016804e29a77fa(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:23,496 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:23,496 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., storeName=f6dcaac235b7e538d527b651e63f9dcc/A, priority=13, startTime=1731808463464; duration=0sec 2024-11-17T01:54:23,496 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:23,496 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6dcaac235b7e538d527b651e63f9dcc:A 2024-11-17T01:54:23,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742382_1558 (size=12207) 2024-11-17T01:54:23,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T01:54:23,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:23,502 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing f6dcaac235b7e538d527b651e63f9dcc 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T01:54:23,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=A 2024-11-17T01:54:23,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:23,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=B 2024-11-17T01:54:23,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:23,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=C 2024-11-17T01:54:23,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:23,502 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/12e4824ce531494ab9b93bc2fba442f8 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/12e4824ce531494ab9b93bc2fba442f8 2024-11-17T01:54:23,509 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6dcaac235b7e538d527b651e63f9dcc/C of f6dcaac235b7e538d527b651e63f9dcc into 12e4824ce531494ab9b93bc2fba442f8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:23,509 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:23,509 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., storeName=f6dcaac235b7e538d527b651e63f9dcc/C, priority=13, startTime=1731808463464; duration=0sec 2024-11-17T01:54:23,509 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:23,509 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6dcaac235b7e538d527b651e63f9dcc:C 2024-11-17T01:54:23,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177121e1e391ed43138b822b57c9c1df28_f6dcaac235b7e538d527b651e63f9dcc is 50, key is test_row_0/A:col10/1731808461875/Put/seqid=0 2024-11-17T01:54:23,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742383_1559 (size=12204) 2024-11-17T01:54:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T01:54:23,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:23,947 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177121e1e391ed43138b822b57c9c1df28_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177121e1e391ed43138b822b57c9c1df28_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:23,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/a3da78494c1b4bad9d2f578d6d2bcec7, store: [table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:23,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/a3da78494c1b4bad9d2f578d6d2bcec7 is 175, key is test_row_0/A:col10/1731808461875/Put/seqid=0 2024-11-17T01:54:23,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742384_1560 (size=31005) 2024-11-17T01:54:23,952 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/a3da78494c1b4bad9d2f578d6d2bcec7 2024-11-17T01:54:23,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/1133af9c4dba4ad18742a84f8ad85370 is 50, key is test_row_0/B:col10/1731808461875/Put/seqid=0 2024-11-17T01:54:23,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742385_1561 (size=12051) 2024-11-17T01:54:23,964 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/1133af9c4dba4ad18742a84f8ad85370 2024-11-17T01:54:23,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/aa04dffc9e844b269a1b51cb052e1b3c is 50, key is test_row_0/C:col10/1731808461875/Put/seqid=0 2024-11-17T01:54:23,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742386_1562 (size=12051) 2024-11-17T01:54:24,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:24,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808524061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:24,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808524174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:24,374 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/aa04dffc9e844b269a1b51cb052e1b3c 2024-11-17T01:54:24,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/a3da78494c1b4bad9d2f578d6d2bcec7 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/a3da78494c1b4bad9d2f578d6d2bcec7 2024-11-17T01:54:24,380 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/a3da78494c1b4bad9d2f578d6d2bcec7, entries=150, sequenceid=132, filesize=30.3 K 2024-11-17T01:54:24,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/1133af9c4dba4ad18742a84f8ad85370 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/1133af9c4dba4ad18742a84f8ad85370 2024-11-17T01:54:24,383 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/1133af9c4dba4ad18742a84f8ad85370, entries=150, sequenceid=132, filesize=11.8 K 2024-11-17T01:54:24,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/aa04dffc9e844b269a1b51cb052e1b3c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/aa04dffc9e844b269a1b51cb052e1b3c 2024-11-17T01:54:24,387 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/aa04dffc9e844b269a1b51cb052e1b3c, entries=150, sequenceid=132, filesize=11.8 K 2024-11-17T01:54:24,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:24,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808524384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:24,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:24,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:24,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:24,388 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f6dcaac235b7e538d527b651e63f9dcc in 886ms, sequenceid=132, compaction requested=false 2024-11-17T01:54:24,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:24,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:24,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-17T01:54:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-17T01:54:24,390 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-17T01:54:24,390 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8020 sec 2024-11-17T01:54:24,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.8060 sec 2024-11-17T01:54:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:24,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6dcaac235b7e538d527b651e63f9dcc 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T01:54:24,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=A 2024-11-17T01:54:24,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:24,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=B 2024-11-17T01:54:24,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:24,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=C 2024-11-17T01:54:24,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:24,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117fa270865afbc4eb5aaf0fe0594b0b786_f6dcaac235b7e538d527b651e63f9dcc is 50, key is test_row_0/A:col10/1731808464035/Put/seqid=0 2024-11-17T01:54:24,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742387_1563 (size=14794) 2024-11-17T01:54:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T01:54:24,734 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:54:24,734 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:54:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-17T01:54:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T01:54:24,736 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:54:24,736 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:54:24,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:54:24,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808524762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T01:54:24,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808524874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:24,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T01:54:24,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:24,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:24,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:24,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:24,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T01:54:25,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:25,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:25,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:25,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T01:54:25,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808525084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:25,106 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:25,108 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117fa270865afbc4eb5aaf0fe0594b0b786_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fa270865afbc4eb5aaf0fe0594b0b786_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:25,109 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/9262bff7cbf24f218c869ed0fdbbe394, store: [table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:25,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/9262bff7cbf24f218c869ed0fdbbe394 is 175, key is test_row_0/A:col10/1731808464035/Put/seqid=0 2024-11-17T01:54:25,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742388_1564 (size=39749) 2024-11-17T01:54:25,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T01:54:25,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:25,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:25,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:25,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T01:54:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:25,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T01:54:25,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:25,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808525394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:25,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:25,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:25,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T01:54:25,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:25,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:25,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:25,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,513 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/9262bff7cbf24f218c869ed0fdbbe394 2024-11-17T01:54:25,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/9805a8bc10264701a981363ef8662de7 is 50, key is test_row_0/B:col10/1731808464035/Put/seqid=0 2024-11-17T01:54:25,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742389_1565 (size=12151) 2024-11-17T01:54:25,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T01:54:25,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:25,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:25,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:25,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T01:54:25,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:25,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:25,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:25,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T01:54:25,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:25,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808525904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:25,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:25,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/9805a8bc10264701a981363ef8662de7 2024-11-17T01:54:25,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/634c48da330f4ee7a1c8380c0d670a79 is 50, key is test_row_0/C:col10/1731808464035/Put/seqid=0 2024-11-17T01:54:25,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742390_1566 (size=12151) 2024-11-17T01:54:25,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T01:54:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:25,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:26,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T01:54:26,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:26,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:26,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:26,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:26,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:26,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T01:54:26,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:26,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:26,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:26,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:26,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:26,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/634c48da330f4ee7a1c8380c0d670a79 2024-11-17T01:54:26,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/9262bff7cbf24f218c869ed0fdbbe394 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/9262bff7cbf24f218c869ed0fdbbe394 2024-11-17T01:54:26,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/9262bff7cbf24f218c869ed0fdbbe394, entries=200, sequenceid=146, filesize=38.8 K 2024-11-17T01:54:26,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/9805a8bc10264701a981363ef8662de7 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/9805a8bc10264701a981363ef8662de7 2024-11-17T01:54:26,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/9805a8bc10264701a981363ef8662de7, entries=150, sequenceid=146, filesize=11.9 K 2024-11-17T01:54:26,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/634c48da330f4ee7a1c8380c0d670a79 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/634c48da330f4ee7a1c8380c0d670a79 2024-11-17T01:54:26,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/634c48da330f4ee7a1c8380c0d670a79, entries=150, sequenceid=146, filesize=11.9 K 2024-11-17T01:54:26,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f6dcaac235b7e538d527b651e63f9dcc in 1644ms, sequenceid=146, compaction requested=true 2024-11-17T01:54:26,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:26,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6dcaac235b7e538d527b651e63f9dcc:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:54:26,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:26,341 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:26,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6dcaac235b7e538d527b651e63f9dcc:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:54:26,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:26,342 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:26,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6dcaac235b7e538d527b651e63f9dcc:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:54:26,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:54:26,342 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:26,342 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): f6dcaac235b7e538d527b651e63f9dcc/B is initiating minor compaction (all files) 2024-11-17T01:54:26,342 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101915 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:26,342 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6dcaac235b7e538d527b651e63f9dcc/B in TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:26,343 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): f6dcaac235b7e538d527b651e63f9dcc/A is initiating minor compaction (all files) 2024-11-17T01:54:26,343 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6dcaac235b7e538d527b651e63f9dcc/A in TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:26,343 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/5d836060fd9e46cba1ef9fc7558592ed, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/1133af9c4dba4ad18742a84f8ad85370, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/9805a8bc10264701a981363ef8662de7] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp, totalSize=35.6 K 2024-11-17T01:54:26,343 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/c9812ccbc00a4f84b0016804e29a77fa, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/a3da78494c1b4bad9d2f578d6d2bcec7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/9262bff7cbf24f218c869ed0fdbbe394] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp, totalSize=99.5 K 2024-11-17T01:54:26,343 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:26,343 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/c9812ccbc00a4f84b0016804e29a77fa, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/a3da78494c1b4bad9d2f578d6d2bcec7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/9262bff7cbf24f218c869ed0fdbbe394] 2024-11-17T01:54:26,344 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d836060fd9e46cba1ef9fc7558592ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731808460652 2024-11-17T01:54:26,344 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9812ccbc00a4f84b0016804e29a77fa, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731808460652 2024-11-17T01:54:26,344 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 1133af9c4dba4ad18742a84f8ad85370, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731808461852 2024-11-17T01:54:26,344 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3da78494c1b4bad9d2f578d6d2bcec7, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731808461852 2024-11-17T01:54:26,345 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 9805a8bc10264701a981363ef8662de7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731808464035 2024-11-17T01:54:26,345 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9262bff7cbf24f218c869ed0fdbbe394, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731808464035 2024-11-17T01:54:26,352 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:26,353 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6dcaac235b7e538d527b651e63f9dcc#B#compaction#478 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:26,353 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/679bde97850a48b7b6f63cb62b14de74 is 50, key is test_row_0/B:col10/1731808464035/Put/seqid=0 2024-11-17T01:54:26,354 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117a11b1fcef3194ff8a17d47680a64c0ee_f6dcaac235b7e538d527b651e63f9dcc store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:26,356 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117a11b1fcef3194ff8a17d47680a64c0ee_f6dcaac235b7e538d527b651e63f9dcc, store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:26,356 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a11b1fcef3194ff8a17d47680a64c0ee_f6dcaac235b7e538d527b651e63f9dcc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:26,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742391_1567 (size=12459) 2024-11-17T01:54:26,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742392_1568 (size=4469) 2024-11-17T01:54:26,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T01:54:26,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:26,413 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing f6dcaac235b7e538d527b651e63f9dcc 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T01:54:26,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=A 2024-11-17T01:54:26,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:26,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=B 2024-11-17T01:54:26,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:26,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=C 2024-11-17T01:54:26,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:26,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117685fe40487a04ff1af23750ed65c9ac4_f6dcaac235b7e538d527b651e63f9dcc is 50, key is test_row_0/A:col10/1731808464761/Put/seqid=0 2024-11-17T01:54:26,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742393_1569 (size=12304) 2024-11-17T01:54:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:26,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:26,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808526748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:26,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808526749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:26,762 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6dcaac235b7e538d527b651e63f9dcc#A#compaction#479 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:26,762 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/ba2f2fe93a484b4c8fb2fde0d014a746 is 175, key is test_row_0/A:col10/1731808464035/Put/seqid=0 2024-11-17T01:54:26,764 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/679bde97850a48b7b6f63cb62b14de74 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/679bde97850a48b7b6f63cb62b14de74 2024-11-17T01:54:26,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57268 deadline: 1731808526764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:26,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57256 deadline: 1731808526764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:26,767 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in f6dcaac235b7e538d527b651e63f9dcc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:26,768 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_0' in f6dcaac235b7e538d527b651e63f9dcc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:26,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742394_1570 (size=31413) 2024-11-17T01:54:26,770 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6dcaac235b7e538d527b651e63f9dcc/B of f6dcaac235b7e538d527b651e63f9dcc into 679bde97850a48b7b6f63cb62b14de74(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:26,770 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:26,770 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., storeName=f6dcaac235b7e538d527b651e63f9dcc/B, priority=13, startTime=1731808466341; duration=0sec 2024-11-17T01:54:26,770 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:54:26,770 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6dcaac235b7e538d527b651e63f9dcc:B 2024-11-17T01:54:26,771 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:26,771 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:26,771 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): f6dcaac235b7e538d527b651e63f9dcc/C is initiating minor compaction (all files) 2024-11-17T01:54:26,771 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6dcaac235b7e538d527b651e63f9dcc/C in TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:26,771 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/12e4824ce531494ab9b93bc2fba442f8, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/aa04dffc9e844b269a1b51cb052e1b3c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/634c48da330f4ee7a1c8380c0d670a79] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp, totalSize=35.6 K 2024-11-17T01:54:26,772 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 12e4824ce531494ab9b93bc2fba442f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731808460652 2024-11-17T01:54:26,773 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting aa04dffc9e844b269a1b51cb052e1b3c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731808461852 2024-11-17T01:54:26,773 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 634c48da330f4ee7a1c8380c0d670a79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731808464035 2024-11-17T01:54:26,779 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6dcaac235b7e538d527b651e63f9dcc#C#compaction#481 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:26,779 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/78efa55ba494483099e4e569e207bfe1 is 50, key is test_row_0/C:col10/1731808464035/Put/seqid=0 2024-11-17T01:54:26,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742395_1571 (size=12459) 2024-11-17T01:54:26,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:26,825 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117685fe40487a04ff1af23750ed65c9ac4_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117685fe40487a04ff1af23750ed65c9ac4_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:26,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/838b03e6a16441a88d1b185d99fe2f6f, store: [table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:26,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/838b03e6a16441a88d1b185d99fe2f6f is 175, key is test_row_0/A:col10/1731808464761/Put/seqid=0 2024-11-17T01:54:26,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742396_1572 (size=31105) 2024-11-17T01:54:26,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808526854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:26,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:26,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:26,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:26,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808526864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T01:54:26,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808526914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:26,916 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in f6dcaac235b7e538d527b651e63f9dcc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:27,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808527064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:27,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:27,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:27,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:27,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808527074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:27,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:27,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:27,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:27,170 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/ba2f2fe93a484b4c8fb2fde0d014a746 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/ba2f2fe93a484b4c8fb2fde0d014a746 2024-11-17T01:54:27,174 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6dcaac235b7e538d527b651e63f9dcc/A of f6dcaac235b7e538d527b651e63f9dcc into ba2f2fe93a484b4c8fb2fde0d014a746(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:27,174 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:27,174 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., storeName=f6dcaac235b7e538d527b651e63f9dcc/A, priority=13, startTime=1731808466341; duration=0sec 2024-11-17T01:54:27,174 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:27,174 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6dcaac235b7e538d527b651e63f9dcc:A 2024-11-17T01:54:27,195 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/78efa55ba494483099e4e569e207bfe1 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/78efa55ba494483099e4e569e207bfe1 2024-11-17T01:54:27,198 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6dcaac235b7e538d527b651e63f9dcc/C of f6dcaac235b7e538d527b651e63f9dcc into 78efa55ba494483099e4e569e207bfe1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:27,198 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:27,198 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., storeName=f6dcaac235b7e538d527b651e63f9dcc/C, priority=13, startTime=1731808466342; duration=0sec 2024-11-17T01:54:27,198 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:27,198 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6dcaac235b7e538d527b651e63f9dcc:C 2024-11-17T01:54:27,229 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/838b03e6a16441a88d1b185d99fe2f6f 2024-11-17T01:54:27,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/c2eaf1e2135249cfa1a944b8be502425 is 50, key is test_row_0/B:col10/1731808464761/Put/seqid=0 2024-11-17T01:54:27,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742397_1573 (size=12151) 2024-11-17T01:54:27,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808527374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:27,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:27,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:27,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:27,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808527384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:27,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:27,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:27,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:27,638 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/c2eaf1e2135249cfa1a944b8be502425 2024-11-17T01:54:27,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/a54b969ddc674191b2f494a47bb751c2 is 50, key is test_row_0/C:col10/1731808464761/Put/seqid=0 2024-11-17T01:54:27,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742398_1574 (size=12151) 2024-11-17T01:54:27,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808527884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:27,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:27,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808527894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:27,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:27,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:27,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:28,049 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/a54b969ddc674191b2f494a47bb751c2 2024-11-17T01:54:28,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/838b03e6a16441a88d1b185d99fe2f6f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/838b03e6a16441a88d1b185d99fe2f6f 2024-11-17T01:54:28,056 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/838b03e6a16441a88d1b185d99fe2f6f, entries=150, sequenceid=165, filesize=30.4 K 2024-11-17T01:54:28,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/c2eaf1e2135249cfa1a944b8be502425 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/c2eaf1e2135249cfa1a944b8be502425 2024-11-17T01:54:28,060 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/c2eaf1e2135249cfa1a944b8be502425, entries=150, sequenceid=165, filesize=11.9 K 2024-11-17T01:54:28,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/a54b969ddc674191b2f494a47bb751c2 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/a54b969ddc674191b2f494a47bb751c2 2024-11-17T01:54:28,064 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/a54b969ddc674191b2f494a47bb751c2, entries=150, sequenceid=165, filesize=11.9 K 2024-11-17T01:54:28,065 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f6dcaac235b7e538d527b651e63f9dcc in 1652ms, sequenceid=165, compaction requested=false 2024-11-17T01:54:28,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:28,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:28,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-17T01:54:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-17T01:54:28,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-17T01:54:28,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3300 sec 2024-11-17T01:54:28,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 3.3340 sec 2024-11-17T01:54:28,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T01:54:28,894 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:54:28,894 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:54:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-17T01:54:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T01:54:28,896 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:54:28,896 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:54:28,896 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:54:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:28,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6dcaac235b7e538d527b651e63f9dcc 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T01:54:28,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=A 2024-11-17T01:54:28,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:28,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=B 2024-11-17T01:54:28,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:28,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=C 2024-11-17T01:54:28,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:28,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117631d8c70dc66405691cf1ac617a7a5bd_f6dcaac235b7e538d527b651e63f9dcc is 50, key is test_row_0/A:col10/1731808466748/Put/seqid=0 2024-11-17T01:54:28,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742399_1575 (size=14794) 2024-11-17T01:54:28,905 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:28,908 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117631d8c70dc66405691cf1ac617a7a5bd_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117631d8c70dc66405691cf1ac617a7a5bd_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:28,908 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/62c3b944a9b1496c9897001ffc75539e, store: [table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:28,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/62c3b944a9b1496c9897001ffc75539e is 175, key is test_row_0/A:col10/1731808466748/Put/seqid=0 2024-11-17T01:54:28,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742400_1576 (size=39749) 2024-11-17T01:54:28,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:28,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808528942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:28,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:28,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:28,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808528944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:28,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808528943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:28,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:28,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:28,948 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in f6dcaac235b7e538d527b651e63f9dcc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:29,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T01:54:29,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T01:54:29,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:29,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:29,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:29,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:29,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:29,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808529054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:29,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808529054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:29,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T01:54:29,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:29,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:29,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:29,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:29,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T01:54:29,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808529264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:29,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808529264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:29,313 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/62c3b944a9b1496c9897001ffc75539e 2024-11-17T01:54:29,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/bbb3631e6cd34afeaa8c62affad6990c is 50, key is test_row_0/B:col10/1731808466748/Put/seqid=0 2024-11-17T01:54:29,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742401_1577 (size=12151) 2024-11-17T01:54:29,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T01:54:29,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:29,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:29,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:29,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:29,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:29,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T01:54:29,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:29,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:29,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:29,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:29,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T01:54:29,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808529574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:29,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808529574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:29,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T01:54:29,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:29,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:29,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:29,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:29,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:29,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/bbb3631e6cd34afeaa8c62affad6990c 2024-11-17T01:54:29,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/56294fa1c5ea4ce08bff6625770db579 is 50, key is test_row_0/C:col10/1731808466748/Put/seqid=0 2024-11-17T01:54:29,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742402_1578 (size=12151) 2024-11-17T01:54:29,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T01:54:29,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:29,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:29,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:29,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:29,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:29,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T01:54:29,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:29,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:29,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:29,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:29,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:29,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T01:54:30,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808530084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:30,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:30,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:30,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:30,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808530094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:30,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T01:54:30,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:30,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:30,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:30,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:30,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:30,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/56294fa1c5ea4ce08bff6625770db579 2024-11-17T01:54:30,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/62c3b944a9b1496c9897001ffc75539e as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/62c3b944a9b1496c9897001ffc75539e 2024-11-17T01:54:30,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/62c3b944a9b1496c9897001ffc75539e, entries=200, sequenceid=182, filesize=38.8 K 2024-11-17T01:54:30,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/bbb3631e6cd34afeaa8c62affad6990c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/bbb3631e6cd34afeaa8c62affad6990c 2024-11-17T01:54:30,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/bbb3631e6cd34afeaa8c62affad6990c, entries=150, sequenceid=182, filesize=11.9 K 2024-11-17T01:54:30,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/56294fa1c5ea4ce08bff6625770db579 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/56294fa1c5ea4ce08bff6625770db579 2024-11-17T01:54:30,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/56294fa1c5ea4ce08bff6625770db579, entries=150, sequenceid=182, filesize=11.9 K 2024-11-17T01:54:30,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f6dcaac235b7e538d527b651e63f9dcc in 1247ms, sequenceid=182, compaction requested=true 2024-11-17T01:54:30,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:30,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6dcaac235b7e538d527b651e63f9dcc:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:54:30,144 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:30,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:30,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6dcaac235b7e538d527b651e63f9dcc:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:54:30,144 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:30,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:30,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6dcaac235b7e538d527b651e63f9dcc:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:54:30,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:54:30,145 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:30,145 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:30,145 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): f6dcaac235b7e538d527b651e63f9dcc/A is initiating minor compaction (all files) 2024-11-17T01:54:30,145 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): f6dcaac235b7e538d527b651e63f9dcc/B is initiating minor compaction (all files) 2024-11-17T01:54:30,146 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6dcaac235b7e538d527b651e63f9dcc/B in TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:30,146 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6dcaac235b7e538d527b651e63f9dcc/A in TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:30,146 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/679bde97850a48b7b6f63cb62b14de74, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/c2eaf1e2135249cfa1a944b8be502425, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/bbb3631e6cd34afeaa8c62affad6990c] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp, totalSize=35.9 K 2024-11-17T01:54:30,146 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/ba2f2fe93a484b4c8fb2fde0d014a746, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/838b03e6a16441a88d1b185d99fe2f6f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/62c3b944a9b1496c9897001ffc75539e] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp, totalSize=99.9 K 2024-11-17T01:54:30,146 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:30,146 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/ba2f2fe93a484b4c8fb2fde0d014a746, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/838b03e6a16441a88d1b185d99fe2f6f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/62c3b944a9b1496c9897001ffc75539e] 2024-11-17T01:54:30,146 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 679bde97850a48b7b6f63cb62b14de74, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731808464035 2024-11-17T01:54:30,146 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba2f2fe93a484b4c8fb2fde0d014a746, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731808464035 2024-11-17T01:54:30,146 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting c2eaf1e2135249cfa1a944b8be502425, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731808464734 2024-11-17T01:54:30,146 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 838b03e6a16441a88d1b185d99fe2f6f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731808464734 2024-11-17T01:54:30,146 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting bbb3631e6cd34afeaa8c62affad6990c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731808466747 2024-11-17T01:54:30,147 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62c3b944a9b1496c9897001ffc75539e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731808466729 2024-11-17T01:54:30,151 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:30,152 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411173167c04d8483476da7674c8905a456bc_f6dcaac235b7e538d527b651e63f9dcc store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:30,152 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6dcaac235b7e538d527b651e63f9dcc#B#compaction#487 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:30,153 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/2d3353a6b92445f38edc665908d8af24 is 50, key is test_row_0/B:col10/1731808466748/Put/seqid=0 2024-11-17T01:54:30,154 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411173167c04d8483476da7674c8905a456bc_f6dcaac235b7e538d527b651e63f9dcc, store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:30,154 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173167c04d8483476da7674c8905a456bc_f6dcaac235b7e538d527b651e63f9dcc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:30,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742403_1579 (size=12561) 2024-11-17T01:54:30,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742404_1580 (size=4469) 2024-11-17T01:54:30,166 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6dcaac235b7e538d527b651e63f9dcc#A#compaction#488 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:30,167 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/7100afe59ff549c58c951e630e92c673 is 175, key is test_row_0/A:col10/1731808466748/Put/seqid=0 2024-11-17T01:54:30,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742405_1581 (size=31515) 2024-11-17T01:54:30,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T01:54:30,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:30,269 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing f6dcaac235b7e538d527b651e63f9dcc 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T01:54:30,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=A 2024-11-17T01:54:30,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:30,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=B 2024-11-17T01:54:30,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:30,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=C 2024-11-17T01:54:30,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:30,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b14e341792e74c1c8b27a24aaf8fabe5_f6dcaac235b7e538d527b651e63f9dcc is 50, key is test_row_0/A:col10/1731808468932/Put/seqid=0 2024-11-17T01:54:30,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742406_1582 (size=12304) 2024-11-17T01:54:30,503 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T01:54:30,560 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/2d3353a6b92445f38edc665908d8af24 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/2d3353a6b92445f38edc665908d8af24 2024-11-17T01:54:30,563 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6dcaac235b7e538d527b651e63f9dcc/B of f6dcaac235b7e538d527b651e63f9dcc into 2d3353a6b92445f38edc665908d8af24(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:30,563 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:30,563 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., storeName=f6dcaac235b7e538d527b651e63f9dcc/B, priority=13, startTime=1731808470144; duration=0sec 2024-11-17T01:54:30,563 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:54:30,563 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6dcaac235b7e538d527b651e63f9dcc:B 2024-11-17T01:54:30,563 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:30,564 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:30,564 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): f6dcaac235b7e538d527b651e63f9dcc/C is initiating minor compaction (all files) 2024-11-17T01:54:30,564 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6dcaac235b7e538d527b651e63f9dcc/C in TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:30,564 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/78efa55ba494483099e4e569e207bfe1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/a54b969ddc674191b2f494a47bb751c2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/56294fa1c5ea4ce08bff6625770db579] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp, totalSize=35.9 K 2024-11-17T01:54:30,564 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 78efa55ba494483099e4e569e207bfe1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731808464035 2024-11-17T01:54:30,565 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting a54b969ddc674191b2f494a47bb751c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731808464734 2024-11-17T01:54:30,565 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 56294fa1c5ea4ce08bff6625770db579, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731808466747 2024-11-17T01:54:30,570 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6dcaac235b7e538d527b651e63f9dcc#C#compaction#490 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:30,571 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/3b707741a0384b978963e9a690321277 is 50, key is test_row_0/C:col10/1731808466748/Put/seqid=0 2024-11-17T01:54:30,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742407_1583 (size=12561) 2024-11-17T01:54:30,579 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/7100afe59ff549c58c951e630e92c673 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/7100afe59ff549c58c951e630e92c673 2024-11-17T01:54:30,582 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6dcaac235b7e538d527b651e63f9dcc/A of f6dcaac235b7e538d527b651e63f9dcc into 7100afe59ff549c58c951e630e92c673(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:30,582 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:30,582 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., storeName=f6dcaac235b7e538d527b651e63f9dcc/A, priority=13, startTime=1731808470144; duration=0sec 2024-11-17T01:54:30,582 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:30,582 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6dcaac235b7e538d527b651e63f9dcc:A 2024-11-17T01:54:30,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:30,692 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b14e341792e74c1c8b27a24aaf8fabe5_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b14e341792e74c1c8b27a24aaf8fabe5_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:30,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/a694ed7c59e24cd8a523e360eef23143, store: [table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:30,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/a694ed7c59e24cd8a523e360eef23143 is 175, key is test_row_0/A:col10/1731808468932/Put/seqid=0 2024-11-17T01:54:30,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742408_1584 (size=31105) 2024-11-17T01:54:30,977 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/3b707741a0384b978963e9a690321277 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/3b707741a0384b978963e9a690321277 2024-11-17T01:54:30,980 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6dcaac235b7e538d527b651e63f9dcc/C of f6dcaac235b7e538d527b651e63f9dcc into 3b707741a0384b978963e9a690321277(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:30,980 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:30,980 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., storeName=f6dcaac235b7e538d527b651e63f9dcc/C, priority=13, startTime=1731808470144; duration=0sec 2024-11-17T01:54:30,981 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:30,981 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6dcaac235b7e538d527b651e63f9dcc:C 2024-11-17T01:54:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T01:54:31,097 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/a694ed7c59e24cd8a523e360eef23143 2024-11-17T01:54:31,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/59a76b27bfb94f2cbe94ff05bb2e0019 is 50, key is test_row_0/B:col10/1731808468932/Put/seqid=0 2024-11-17T01:54:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:31,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:31,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742409_1585 (size=12151) 2024-11-17T01:54:31,113 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/59a76b27bfb94f2cbe94ff05bb2e0019 2024-11-17T01:54:31,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/da32811eeed64f3fa49b521f813fcf06 is 50, key is test_row_0/C:col10/1731808468932/Put/seqid=0 2024-11-17T01:54:31,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742410_1586 (size=12151) 2024-11-17T01:54:31,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808531134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:31,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:31,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:31,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:31,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808531134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:31,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:31,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:31,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:31,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808531244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:31,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808531244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:31,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:31,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:31,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:31,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:31,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:31,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:31,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808531454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:31,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808531454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:31,529 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/da32811eeed64f3fa49b521f813fcf06 2024-11-17T01:54:31,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/a694ed7c59e24cd8a523e360eef23143 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/a694ed7c59e24cd8a523e360eef23143 2024-11-17T01:54:31,534 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/a694ed7c59e24cd8a523e360eef23143, entries=150, sequenceid=200, filesize=30.4 K 2024-11-17T01:54:31,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/59a76b27bfb94f2cbe94ff05bb2e0019 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/59a76b27bfb94f2cbe94ff05bb2e0019 2024-11-17T01:54:31,537 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/59a76b27bfb94f2cbe94ff05bb2e0019, entries=150, sequenceid=200, filesize=11.9 K 2024-11-17T01:54:31,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/da32811eeed64f3fa49b521f813fcf06 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/da32811eeed64f3fa49b521f813fcf06 2024-11-17T01:54:31,541 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/da32811eeed64f3fa49b521f813fcf06, entries=150, sequenceid=200, filesize=11.9 K 2024-11-17T01:54:31,542 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f6dcaac235b7e538d527b651e63f9dcc in 1272ms, sequenceid=200, compaction requested=false 2024-11-17T01:54:31,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:31,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:31,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-17T01:54:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-17T01:54:31,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-17T01:54:31,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6470 sec 2024-11-17T01:54:31,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.6510 sec 2024-11-17T01:54:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:31,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6dcaac235b7e538d527b651e63f9dcc 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T01:54:31,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=A 2024-11-17T01:54:31,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:31,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=B 2024-11-17T01:54:31,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:31,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=C 2024-11-17T01:54:31,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:31,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117fe9f16122b9a48ed81ce09ef719b235f_f6dcaac235b7e538d527b651e63f9dcc is 50, key is test_row_0/A:col10/1731808471110/Put/seqid=0 2024-11-17T01:54:31,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742411_1587 (size=12304) 2024-11-17T01:54:31,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:31,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808531800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:31,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:31,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:31,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:31,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:31,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808531804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:31,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:31,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808531914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:31,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:31,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808531914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:32,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808532123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:32,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:32,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:32,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808532124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:32,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:32,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:32,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:32,180 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:32,183 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117fe9f16122b9a48ed81ce09ef719b235f_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fe9f16122b9a48ed81ce09ef719b235f_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:32,184 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/66458c026bb04f2db5facc397d55a136, store: [table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:32,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/66458c026bb04f2db5facc397d55a136 is 175, key is test_row_0/A:col10/1731808471110/Put/seqid=0 2024-11-17T01:54:32,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742412_1588 (size=31105) 2024-11-17T01:54:32,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808532434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:32,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808532434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:32,587 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/66458c026bb04f2db5facc397d55a136 2024-11-17T01:54:32,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/ccd85ff9674c4dd98856e251ccb86c2f is 50, key is test_row_0/B:col10/1731808471110/Put/seqid=0 2024-11-17T01:54:32,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742413_1589 (size=12151) 2024-11-17T01:54:32,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808532944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:32,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:32,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:32,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:32,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808532944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:32,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:32,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:32,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:32,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57238 deadline: 1731808532964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:32,967 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c6c0d7a0a7c0,41157,1731808292173 for 'test_row_1' in f6dcaac235b7e538d527b651e63f9dcc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T01:54:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:32,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/ccd85ff9674c4dd98856e251ccb86c2f 2024-11-17T01:54:33,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/92ca2f2b29b547aea292b90d6680332c is 50, key is test_row_0/C:col10/1731808471110/Put/seqid=0 2024-11-17T01:54:33,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742414_1590 (size=12151) 2024-11-17T01:54:33,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T01:54:33,054 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:54:33,055 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T01:54:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-17T01:54:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T01:54:33,056 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T01:54:33,057 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T01:54:33,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T01:54:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T01:54:33,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T01:54:33,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:33,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:33,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:33,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:33,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:33,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T01:54:33,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:33,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:33,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:33,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:33,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:33,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:54:33,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T01:54:33,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/92ca2f2b29b547aea292b90d6680332c 2024-11-17T01:54:33,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/66458c026bb04f2db5facc397d55a136 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/66458c026bb04f2db5facc397d55a136 2024-11-17T01:54:33,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/66458c026bb04f2db5facc397d55a136, entries=150, sequenceid=218, filesize=30.4 K 2024-11-17T01:54:33,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/ccd85ff9674c4dd98856e251ccb86c2f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/ccd85ff9674c4dd98856e251ccb86c2f 2024-11-17T01:54:33,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/ccd85ff9674c4dd98856e251ccb86c2f, entries=150, sequenceid=218, filesize=11.9 K 2024-11-17T01:54:33,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/92ca2f2b29b547aea292b90d6680332c as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/92ca2f2b29b547aea292b90d6680332c 2024-11-17T01:54:33,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/92ca2f2b29b547aea292b90d6680332c, entries=150, sequenceid=218, filesize=11.9 K 2024-11-17T01:54:33,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f6dcaac235b7e538d527b651e63f9dcc in 1652ms, sequenceid=218, compaction requested=true 2024-11-17T01:54:33,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:33,420 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:33,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6dcaac235b7e538d527b651e63f9dcc:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:54:33,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:33,420 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:33,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6dcaac235b7e538d527b651e63f9dcc:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:54:33,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:33,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6dcaac235b7e538d527b651e63f9dcc:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:54:33,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:54:33,420 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:33,420 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): f6dcaac235b7e538d527b651e63f9dcc/A is initiating minor compaction (all files) 2024-11-17T01:54:33,420 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6dcaac235b7e538d527b651e63f9dcc/A in TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:33,421 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/7100afe59ff549c58c951e630e92c673, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/a694ed7c59e24cd8a523e360eef23143, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/66458c026bb04f2db5facc397d55a136] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp, totalSize=91.5 K 2024-11-17T01:54:33,421 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:33,421 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/7100afe59ff549c58c951e630e92c673, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/a694ed7c59e24cd8a523e360eef23143, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/66458c026bb04f2db5facc397d55a136] 2024-11-17T01:54:33,421 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:33,421 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): f6dcaac235b7e538d527b651e63f9dcc/B is initiating minor compaction (all files) 2024-11-17T01:54:33,421 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6dcaac235b7e538d527b651e63f9dcc/B in TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:33,421 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7100afe59ff549c58c951e630e92c673, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731808466747 2024-11-17T01:54:33,421 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/2d3353a6b92445f38edc665908d8af24, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/59a76b27bfb94f2cbe94ff05bb2e0019, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/ccd85ff9674c4dd98856e251ccb86c2f] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp, totalSize=36.0 K 2024-11-17T01:54:33,421 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting a694ed7c59e24cd8a523e360eef23143, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731808468932 2024-11-17T01:54:33,421 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d3353a6b92445f38edc665908d8af24, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731808466747 2024-11-17T01:54:33,421 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66458c026bb04f2db5facc397d55a136, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731808471110 2024-11-17T01:54:33,421 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 59a76b27bfb94f2cbe94ff05bb2e0019, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731808468932 2024-11-17T01:54:33,422 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting ccd85ff9674c4dd98856e251ccb86c2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731808471110 2024-11-17T01:54:33,425 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:33,427 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117be896db491364be6a30a4573899d9cab_f6dcaac235b7e538d527b651e63f9dcc store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:33,427 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6dcaac235b7e538d527b651e63f9dcc#B#compaction#496 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:33,427 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/0abfb768b7c94a0d8f2f93ca8ac3ddeb is 50, key is test_row_0/B:col10/1731808471110/Put/seqid=0 2024-11-17T01:54:33,428 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117be896db491364be6a30a4573899d9cab_f6dcaac235b7e538d527b651e63f9dcc, store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:33,428 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117be896db491364be6a30a4573899d9cab_f6dcaac235b7e538d527b651e63f9dcc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:33,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742415_1591 (size=12663) 2024-11-17T01:54:33,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742416_1592 (size=4469) 2024-11-17T01:54:33,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41157 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T01:54:33,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:33,514 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing f6dcaac235b7e538d527b651e63f9dcc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T01:54:33,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=A 2024-11-17T01:54:33,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:33,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=B 2024-11-17T01:54:33,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:33,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=C 2024-11-17T01:54:33,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:33,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117994b69b6c4774e26abd6966c33562dfa_f6dcaac235b7e538d527b651e63f9dcc is 50, key is test_row_0/A:col10/1731808471799/Put/seqid=0 2024-11-17T01:54:33,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742417_1593 (size=12304) 2024-11-17T01:54:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T01:54:33,832 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6dcaac235b7e538d527b651e63f9dcc#A#compaction#497 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:33,833 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/5c758def34c04bbba9e479943a8b4700 is 175, key is test_row_0/A:col10/1731808471110/Put/seqid=0 2024-11-17T01:54:33,840 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/0abfb768b7c94a0d8f2f93ca8ac3ddeb as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/0abfb768b7c94a0d8f2f93ca8ac3ddeb 2024-11-17T01:54:33,844 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6dcaac235b7e538d527b651e63f9dcc/B of f6dcaac235b7e538d527b651e63f9dcc into 0abfb768b7c94a0d8f2f93ca8ac3ddeb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:33,844 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:33,844 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., storeName=f6dcaac235b7e538d527b651e63f9dcc/B, priority=13, startTime=1731808473420; duration=0sec 2024-11-17T01:54:33,844 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:54:33,844 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6dcaac235b7e538d527b651e63f9dcc:B 2024-11-17T01:54:33,844 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:33,845 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:33,845 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): f6dcaac235b7e538d527b651e63f9dcc/C is initiating minor compaction (all files) 2024-11-17T01:54:33,845 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6dcaac235b7e538d527b651e63f9dcc/C in TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:33,845 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/3b707741a0384b978963e9a690321277, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/da32811eeed64f3fa49b521f813fcf06, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/92ca2f2b29b547aea292b90d6680332c] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp, totalSize=36.0 K 2024-11-17T01:54:33,845 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b707741a0384b978963e9a690321277, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731808466747 2024-11-17T01:54:33,845 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting da32811eeed64f3fa49b521f813fcf06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731808468932 2024-11-17T01:54:33,846 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 92ca2f2b29b547aea292b90d6680332c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731808471110 2024-11-17T01:54:33,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742418_1594 (size=31617) 2024-11-17T01:54:33,850 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/5c758def34c04bbba9e479943a8b4700 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/5c758def34c04bbba9e479943a8b4700 2024-11-17T01:54:33,854 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6dcaac235b7e538d527b651e63f9dcc/A of f6dcaac235b7e538d527b651e63f9dcc into 5c758def34c04bbba9e479943a8b4700(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:33,854 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:33,854 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., storeName=f6dcaac235b7e538d527b651e63f9dcc/A, priority=13, startTime=1731808473420; duration=0sec 2024-11-17T01:54:33,855 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:33,855 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6dcaac235b7e538d527b651e63f9dcc:A 2024-11-17T01:54:33,861 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6dcaac235b7e538d527b651e63f9dcc#C#compaction#499 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:33,861 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/9876425c602048cc909a73aaf7dffeab is 50, key is test_row_0/C:col10/1731808471110/Put/seqid=0 2024-11-17T01:54:33,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742419_1595 (size=12663) 2024-11-17T01:54:33,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:33,925 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117994b69b6c4774e26abd6966c33562dfa_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117994b69b6c4774e26abd6966c33562dfa_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:33,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/cba06c5daeb44be6a3f72651e97571bd, store: [table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:33,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/cba06c5daeb44be6a3f72651e97571bd is 175, key is test_row_0/A:col10/1731808471799/Put/seqid=0 2024-11-17T01:54:33,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742420_1596 (size=31105) 2024-11-17T01:54:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:33,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. as already flushing 2024-11-17T01:54:34,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:34,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808533997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:34,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:34,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808534003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:34,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:34,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808534114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:34,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:34,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808534114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:34,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T01:54:34,258 INFO [Thread-2385 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2385. 2024-11-17T01:54:34,258 DEBUG [Thread-2385 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:54:34,258 DEBUG [Thread-2385 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:34,258 DEBUG [Thread-2385 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:34,258 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:34,259 INFO [Thread-2383 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2383. 2024-11-17T01:54:34,259 DEBUG [Thread-2383 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:54:34,259 DEBUG [Thread-2383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:34,259 DEBUG [Thread-2383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:34,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:34,259 INFO [Thread-2377 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2377. 2024-11-17T01:54:34,259 DEBUG [Thread-2377 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:54:34,259 DEBUG [Thread-2377 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:34,259 DEBUG [Thread-2377 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:34,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:34,262 INFO [Thread-2379 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2379. 2024-11-17T01:54:34,262 DEBUG [Thread-2379 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:54:34,262 DEBUG [Thread-2379 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:34,262 DEBUG [Thread-2379 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:34,262 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:34,262 INFO [Thread-2381 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2381. 2024-11-17T01:54:34,262 DEBUG [Thread-2381 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:54:34,262 DEBUG [Thread-2381 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:34,262 DEBUG [Thread-2381 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:34,262 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:34,268 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/9876425c602048cc909a73aaf7dffeab as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/9876425c602048cc909a73aaf7dffeab 2024-11-17T01:54:34,271 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6dcaac235b7e538d527b651e63f9dcc/C of f6dcaac235b7e538d527b651e63f9dcc into 9876425c602048cc909a73aaf7dffeab(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:34,271 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:34,271 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., storeName=f6dcaac235b7e538d527b651e63f9dcc/C, priority=13, startTime=1731808473420; duration=0sec 2024-11-17T01:54:34,271 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:34,271 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6dcaac235b7e538d527b651e63f9dcc:C 2024-11-17T01:54:34,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:34,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:34,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808534325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:34,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808534325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:34,329 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/cba06c5daeb44be6a3f72651e97571bd 2024-11-17T01:54:34,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/1e84cd0b4d674736a468d71c2caa8774 is 50, key is test_row_0/B:col10/1731808471799/Put/seqid=0 2024-11-17T01:54:34,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742421_1597 (size=12151) 2024-11-17T01:54:34,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:34,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808534635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808534635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:34,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:34,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:34,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:34,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:34,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:34,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:34,745 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/1e84cd0b4d674736a468d71c2caa8774 2024-11-17T01:54:34,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/49515d0c8c544d28bee3320e5f0811e1 is 50, key is test_row_0/C:col10/1731808471799/Put/seqid=0 2024-11-17T01:54:34,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742422_1598 (size=12151) 2024-11-17T01:54:35,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:35,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T01:54:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57230 deadline: 1731808535145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57220 deadline: 1731808535145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:35,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 , the old value is region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T01:54:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., hostname=c6c0d7a0a7c0,41157,1731808292173, seqNum=5 because the exception is null or not the one we care about 2024-11-17T01:54:35,160 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/49515d0c8c544d28bee3320e5f0811e1 2024-11-17T01:54:35,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/cba06c5daeb44be6a3f72651e97571bd as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/cba06c5daeb44be6a3f72651e97571bd 2024-11-17T01:54:35,173 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/cba06c5daeb44be6a3f72651e97571bd, entries=150, sequenceid=235, filesize=30.4 K 2024-11-17T01:54:35,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/1e84cd0b4d674736a468d71c2caa8774 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/1e84cd0b4d674736a468d71c2caa8774 2024-11-17T01:54:35,177 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/1e84cd0b4d674736a468d71c2caa8774, entries=150, sequenceid=235, filesize=11.9 K 2024-11-17T01:54:35,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/49515d0c8c544d28bee3320e5f0811e1 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/49515d0c8c544d28bee3320e5f0811e1 2024-11-17T01:54:35,180 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/49515d0c8c544d28bee3320e5f0811e1, entries=150, sequenceid=235, filesize=11.9 K 2024-11-17T01:54:35,181 INFO [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f6dcaac235b7e538d527b651e63f9dcc in 1667ms, sequenceid=235, compaction requested=false 2024-11-17T01:54:35,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:35,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:35,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-17T01:54:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-17T01:54:35,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-17T01:54:35,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1260 sec 2024-11-17T01:54:35,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.1300 sec 2024-11-17T01:54:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T01:54:35,205 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:54:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41157 {}] regionserver.HRegion(8855): Flush requested on f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:36,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6dcaac235b7e538d527b651e63f9dcc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T01:54:36,158 INFO [Thread-2370 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2370. 2024-11-17T01:54:36,158 INFO [Thread-2374 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2374. 2024-11-17T01:54:36,158 DEBUG [Thread-2370 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:54:36,158 DEBUG [Thread-2370 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:36,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=A 2024-11-17T01:54:36,158 DEBUG [Thread-2374 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:54:36,158 DEBUG [Thread-2374 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:36,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:36,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=B 2024-11-17T01:54:36,159 DEBUG [Thread-2370 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:36,159 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:36,159 DEBUG [Thread-2374 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:36,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:36,159 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:36,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=C 2024-11-17T01:54:36,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:36,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111709a6ab44f60c46408ce126002ce826c6_f6dcaac235b7e538d527b651e63f9dcc is 50, key is test_row_0/A:col10/1731808476155/Put/seqid=0 2024-11-17T01:54:36,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742423_1599 (size=12304) 2024-11-17T01:54:36,572 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:36,578 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111709a6ab44f60c46408ce126002ce826c6_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111709a6ab44f60c46408ce126002ce826c6_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:36,579 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/43c1c2d9e7df413793d0da7fb010a8e0, store: [table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:36,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/43c1c2d9e7df413793d0da7fb010a8e0 is 175, key is test_row_0/A:col10/1731808476155/Put/seqid=0 2024-11-17T01:54:36,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742424_1600 (size=31105) 2024-11-17T01:54:36,828 INFO [Thread-2372 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2372. 2024-11-17T01:54:36,828 DEBUG [Thread-2372 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:54:36,828 DEBUG [Thread-2372 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:36,828 DEBUG [Thread-2372 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:36,828 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:36,848 INFO [Thread-2368 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2368. 2024-11-17T01:54:36,848 DEBUG [Thread-2368 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:54:36,848 DEBUG [Thread-2368 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:36,849 DEBUG [Thread-2368 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:36,849 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:36,985 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/43c1c2d9e7df413793d0da7fb010a8e0 2024-11-17T01:54:36,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/e1639ee8a4e74cfc853421d4bfd61d96 is 50, key is test_row_0/B:col10/1731808476155/Put/seqid=0 2024-11-17T01:54:37,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742425_1601 (size=12151) 2024-11-17T01:54:37,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/e1639ee8a4e74cfc853421d4bfd61d96 2024-11-17T01:54:37,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/bdfddfa3a65e45d281d57366a3284159 is 50, key is test_row_0/C:col10/1731808476155/Put/seqid=0 2024-11-17T01:54:37,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742426_1602 (size=12151) 2024-11-17T01:54:37,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/bdfddfa3a65e45d281d57366a3284159 2024-11-17T01:54:37,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/43c1c2d9e7df413793d0da7fb010a8e0 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/43c1c2d9e7df413793d0da7fb010a8e0 2024-11-17T01:54:37,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/43c1c2d9e7df413793d0da7fb010a8e0, entries=150, sequenceid=255, filesize=30.4 K 2024-11-17T01:54:37,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/e1639ee8a4e74cfc853421d4bfd61d96 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/e1639ee8a4e74cfc853421d4bfd61d96 2024-11-17T01:54:37,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/e1639ee8a4e74cfc853421d4bfd61d96, entries=150, sequenceid=255, filesize=11.9 K 2024-11-17T01:54:37,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/bdfddfa3a65e45d281d57366a3284159 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/bdfddfa3a65e45d281d57366a3284159 2024-11-17T01:54:37,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/bdfddfa3a65e45d281d57366a3284159, entries=150, sequenceid=255, filesize=11.9 K 2024-11-17T01:54:37,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=13.42 KB/13740 for f6dcaac235b7e538d527b651e63f9dcc in 1679ms, sequenceid=255, compaction requested=true 2024-11-17T01:54:37,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:37,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6dcaac235b7e538d527b651e63f9dcc:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T01:54:37,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:37,838 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:37,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6dcaac235b7e538d527b651e63f9dcc:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T01:54:37,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:37,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6dcaac235b7e538d527b651e63f9dcc:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T01:54:37,838 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:37,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:54:37,839 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:37,839 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:37,839 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1541): f6dcaac235b7e538d527b651e63f9dcc/A is initiating minor compaction (all files) 2024-11-17T01:54:37,839 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): f6dcaac235b7e538d527b651e63f9dcc/B is initiating minor compaction (all files) 2024-11-17T01:54:37,839 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6dcaac235b7e538d527b651e63f9dcc/A in TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:37,839 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6dcaac235b7e538d527b651e63f9dcc/B in TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:37,839 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/5c758def34c04bbba9e479943a8b4700, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/cba06c5daeb44be6a3f72651e97571bd, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/43c1c2d9e7df413793d0da7fb010a8e0] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp, totalSize=91.6 K 2024-11-17T01:54:37,839 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/0abfb768b7c94a0d8f2f93ca8ac3ddeb, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/1e84cd0b4d674736a468d71c2caa8774, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/e1639ee8a4e74cfc853421d4bfd61d96] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp, totalSize=36.1 K 2024-11-17T01:54:37,839 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:37,839 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. files: [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/5c758def34c04bbba9e479943a8b4700, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/cba06c5daeb44be6a3f72651e97571bd, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/43c1c2d9e7df413793d0da7fb010a8e0] 2024-11-17T01:54:37,840 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 0abfb768b7c94a0d8f2f93ca8ac3ddeb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731808471110 2024-11-17T01:54:37,840 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c758def34c04bbba9e479943a8b4700, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731808471110 2024-11-17T01:54:37,840 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e84cd0b4d674736a468d71c2caa8774, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731808471788 2024-11-17T01:54:37,840 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting cba06c5daeb44be6a3f72651e97571bd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731808471788 2024-11-17T01:54:37,840 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting e1639ee8a4e74cfc853421d4bfd61d96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731808473968 2024-11-17T01:54:37,841 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43c1c2d9e7df413793d0da7fb010a8e0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731808473968 2024-11-17T01:54:37,847 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:37,849 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6dcaac235b7e538d527b651e63f9dcc#B#compaction#505 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:37,849 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411173710205daa9b493399967823f24e8f45_f6dcaac235b7e538d527b651e63f9dcc store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:37,849 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/5f63df939cce44098fa4ca820f2fa287 is 50, key is test_row_0/B:col10/1731808476155/Put/seqid=0 2024-11-17T01:54:37,852 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411173710205daa9b493399967823f24e8f45_f6dcaac235b7e538d527b651e63f9dcc, store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:37,852 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173710205daa9b493399967823f24e8f45_f6dcaac235b7e538d527b651e63f9dcc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:37,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742427_1603 (size=12765) 2024-11-17T01:54:37,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742428_1604 (size=4469) 2024-11-17T01:54:38,256 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6dcaac235b7e538d527b651e63f9dcc#A#compaction#506 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:38,257 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/f50067bf51ed4903a86e0e763e4c610f is 175, key is test_row_0/A:col10/1731808476155/Put/seqid=0 2024-11-17T01:54:38,258 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/5f63df939cce44098fa4ca820f2fa287 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/5f63df939cce44098fa4ca820f2fa287 2024-11-17T01:54:38,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742429_1605 (size=31719) 2024-11-17T01:54:38,262 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6dcaac235b7e538d527b651e63f9dcc/B of f6dcaac235b7e538d527b651e63f9dcc into 5f63df939cce44098fa4ca820f2fa287(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:38,262 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:38,262 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., storeName=f6dcaac235b7e538d527b651e63f9dcc/B, priority=13, startTime=1731808477838; duration=0sec 2024-11-17T01:54:38,262 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T01:54:38,262 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6dcaac235b7e538d527b651e63f9dcc:B 2024-11-17T01:54:38,262 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T01:54:38,263 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T01:54:38,263 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1541): f6dcaac235b7e538d527b651e63f9dcc/C is initiating minor compaction (all files) 2024-11-17T01:54:38,263 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6dcaac235b7e538d527b651e63f9dcc/C in TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:38,263 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/9876425c602048cc909a73aaf7dffeab, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/49515d0c8c544d28bee3320e5f0811e1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/bdfddfa3a65e45d281d57366a3284159] into tmpdir=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp, totalSize=36.1 K 2024-11-17T01:54:38,263 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 9876425c602048cc909a73aaf7dffeab, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731808471110 2024-11-17T01:54:38,264 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting 49515d0c8c544d28bee3320e5f0811e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731808471788 2024-11-17T01:54:38,264 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] compactions.Compactor(225): Compacting bdfddfa3a65e45d281d57366a3284159, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731808473968 2024-11-17T01:54:38,271 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6dcaac235b7e538d527b651e63f9dcc#C#compaction#507 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T01:54:38,272 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/38fe0a33b473446f99fbddf2f173c33f is 50, key is test_row_0/C:col10/1731808476155/Put/seqid=0 2024-11-17T01:54:38,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742430_1606 (size=12765) 2024-11-17T01:54:38,665 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/f50067bf51ed4903a86e0e763e4c610f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/f50067bf51ed4903a86e0e763e4c610f 2024-11-17T01:54:38,671 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6dcaac235b7e538d527b651e63f9dcc/A of f6dcaac235b7e538d527b651e63f9dcc into f50067bf51ed4903a86e0e763e4c610f(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:38,671 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:38,671 INFO [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., storeName=f6dcaac235b7e538d527b651e63f9dcc/A, priority=13, startTime=1731808477838; duration=0sec 2024-11-17T01:54:38,671 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:38,671 DEBUG [RS:0;c6c0d7a0a7c0:41157-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6dcaac235b7e538d527b651e63f9dcc:A 2024-11-17T01:54:38,680 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/38fe0a33b473446f99fbddf2f173c33f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/38fe0a33b473446f99fbddf2f173c33f 2024-11-17T01:54:38,687 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6dcaac235b7e538d527b651e63f9dcc/C of f6dcaac235b7e538d527b651e63f9dcc into 38fe0a33b473446f99fbddf2f173c33f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T01:54:38,687 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6dcaac235b7e538d527b651e63f9dcc: 2024-11-17T01:54:38,687 INFO [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., storeName=f6dcaac235b7e538d527b651e63f9dcc/C, priority=13, startTime=1731808477838; duration=0sec 2024-11-17T01:54:38,687 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T01:54:38,687 DEBUG [RS:0;c6c0d7a0a7c0:41157-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6dcaac235b7e538d527b651e63f9dcc:C 2024-11-17T01:54:43,056 INFO [Thread-2366 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2366. 2024-11-17T01:54:43,056 DEBUG [Thread-2366 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T01:54:43,056 DEBUG [Thread-2366 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:43,056 DEBUG [Thread-2366 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:43,056 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:43,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T01:54:43,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-17T01:54:43,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-17T01:54:43,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-17T01:54:43,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-17T01:54:43,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-17T01:54:43,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T01:54:43,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3407 2024-11-17T01:54:43,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3373 2024-11-17T01:54:43,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3380 2024-11-17T01:54:43,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3409 2024-11-17T01:54:43,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3375 2024-11-17T01:54:43,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T01:54:43,057 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T01:54:43,057 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:54:43,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:43,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:43,057 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:43,058 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T01:54:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T01:54:43,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T01:54:43,062 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808483062"}]},"ts":"1731808483062"} 2024-11-17T01:54:43,064 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T01:54:43,064 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T01:54:43,065 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T01:54:43,067 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6dcaac235b7e538d527b651e63f9dcc, UNASSIGN}] 2024-11-17T01:54:43,068 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6dcaac235b7e538d527b651e63f9dcc, UNASSIGN 2024-11-17T01:54:43,069 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=f6dcaac235b7e538d527b651e63f9dcc, regionState=CLOSING, regionLocation=c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:43,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6dcaac235b7e538d527b651e63f9dcc, UNASSIGN because future has completed 2024-11-17T01:54:43,071 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T01:54:43,072 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173}] 2024-11-17T01:54:43,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T01:54:43,225 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:43,225 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T01:54:43,226 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing f6dcaac235b7e538d527b651e63f9dcc, disabling compactions & flushes 2024-11-17T01:54:43,226 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:43,226 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:43,226 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. after waiting 0 ms 2024-11-17T01:54:43,226 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:43,226 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing f6dcaac235b7e538d527b651e63f9dcc 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-17T01:54:43,227 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=A 2024-11-17T01:54:43,227 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:43,227 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=B 2024-11-17T01:54:43,227 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:43,227 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f6dcaac235b7e538d527b651e63f9dcc, store=C 2024-11-17T01:54:43,227 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T01:54:43,237 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173916334a3030468794f74380c8078637_f6dcaac235b7e538d527b651e63f9dcc is 50, key is test_row_0/A:col10/1731808476845/Put/seqid=0 2024-11-17T01:54:43,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742431_1607 (size=9914) 2024-11-17T01:54:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T01:54:43,641 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:54:43,643 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173916334a3030468794f74380c8078637_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173916334a3030468794f74380c8078637_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:43,644 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/0ed46852ba0a41809dd48a0df60cfca3, store: [table=TestAcidGuarantees family=A region=f6dcaac235b7e538d527b651e63f9dcc] 2024-11-17T01:54:43,644 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/0ed46852ba0a41809dd48a0df60cfca3 is 175, key is test_row_0/A:col10/1731808476845/Put/seqid=0 2024-11-17T01:54:43,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742432_1608 (size=22561) 2024-11-17T01:54:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T01:54:44,048 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/0ed46852ba0a41809dd48a0df60cfca3 2024-11-17T01:54:44,058 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/ee416d0bd89b43108dc03593d1e47cfa is 50, key is test_row_0/B:col10/1731808476845/Put/seqid=0 2024-11-17T01:54:44,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742433_1609 (size=9857) 2024-11-17T01:54:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T01:54:44,463 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/ee416d0bd89b43108dc03593d1e47cfa 2024-11-17T01:54:44,476 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/45be18af0a664cd6a498c7302afc5d9f is 50, key is test_row_0/C:col10/1731808476845/Put/seqid=0 2024-11-17T01:54:44,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742434_1610 (size=9857) 2024-11-17T01:54:44,881 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/45be18af0a664cd6a498c7302afc5d9f 2024-11-17T01:54:44,888 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/A/0ed46852ba0a41809dd48a0df60cfca3 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/0ed46852ba0a41809dd48a0df60cfca3 2024-11-17T01:54:44,894 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/0ed46852ba0a41809dd48a0df60cfca3, entries=100, sequenceid=264, filesize=22.0 K 2024-11-17T01:54:44,895 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/B/ee416d0bd89b43108dc03593d1e47cfa as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/ee416d0bd89b43108dc03593d1e47cfa 2024-11-17T01:54:44,900 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/ee416d0bd89b43108dc03593d1e47cfa, entries=100, sequenceid=264, filesize=9.6 K 2024-11-17T01:54:44,901 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/.tmp/C/45be18af0a664cd6a498c7302afc5d9f as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/45be18af0a664cd6a498c7302afc5d9f 2024-11-17T01:54:44,906 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/45be18af0a664cd6a498c7302afc5d9f, entries=100, sequenceid=264, filesize=9.6 K 2024-11-17T01:54:44,907 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for f6dcaac235b7e538d527b651e63f9dcc in 1680ms, sequenceid=264, compaction requested=false 2024-11-17T01:54:44,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/35c017629e1849ffadc07f3091a620e1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/86d1cee2d8014cf3a21427f64320cc47, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/1866a8f327104812ba09e3590103d1d2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/df46b6d2b8e041b0b5e3f18c1d729b4a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/342fce7fb08e4e378d757bfc72da8838, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/e7090818afc24abd940be41c1dc35c2d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/c9812ccbc00a4f84b0016804e29a77fa, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/a3da78494c1b4bad9d2f578d6d2bcec7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/9262bff7cbf24f218c869ed0fdbbe394, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/ba2f2fe93a484b4c8fb2fde0d014a746, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/838b03e6a16441a88d1b185d99fe2f6f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/62c3b944a9b1496c9897001ffc75539e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/7100afe59ff549c58c951e630e92c673, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/a694ed7c59e24cd8a523e360eef23143, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/5c758def34c04bbba9e479943a8b4700, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/66458c026bb04f2db5facc397d55a136, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/cba06c5daeb44be6a3f72651e97571bd, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/43c1c2d9e7df413793d0da7fb010a8e0] to archive 2024-11-17T01:54:44,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T01:54:44,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/35c017629e1849ffadc07f3091a620e1 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/35c017629e1849ffadc07f3091a620e1 2024-11-17T01:54:44,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/86d1cee2d8014cf3a21427f64320cc47 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/86d1cee2d8014cf3a21427f64320cc47 2024-11-17T01:54:44,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/1866a8f327104812ba09e3590103d1d2 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/1866a8f327104812ba09e3590103d1d2 2024-11-17T01:54:44,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/df46b6d2b8e041b0b5e3f18c1d729b4a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/df46b6d2b8e041b0b5e3f18c1d729b4a 2024-11-17T01:54:44,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/342fce7fb08e4e378d757bfc72da8838 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/342fce7fb08e4e378d757bfc72da8838 2024-11-17T01:54:44,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/e7090818afc24abd940be41c1dc35c2d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/e7090818afc24abd940be41c1dc35c2d 2024-11-17T01:54:44,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/c9812ccbc00a4f84b0016804e29a77fa to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/c9812ccbc00a4f84b0016804e29a77fa 2024-11-17T01:54:44,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/a3da78494c1b4bad9d2f578d6d2bcec7 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/a3da78494c1b4bad9d2f578d6d2bcec7 2024-11-17T01:54:44,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/9262bff7cbf24f218c869ed0fdbbe394 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/9262bff7cbf24f218c869ed0fdbbe394 2024-11-17T01:54:44,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/ba2f2fe93a484b4c8fb2fde0d014a746 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/ba2f2fe93a484b4c8fb2fde0d014a746 2024-11-17T01:54:44,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/838b03e6a16441a88d1b185d99fe2f6f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/838b03e6a16441a88d1b185d99fe2f6f 2024-11-17T01:54:44,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/62c3b944a9b1496c9897001ffc75539e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/62c3b944a9b1496c9897001ffc75539e 2024-11-17T01:54:44,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/7100afe59ff549c58c951e630e92c673 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/7100afe59ff549c58c951e630e92c673 2024-11-17T01:54:44,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/a694ed7c59e24cd8a523e360eef23143 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/a694ed7c59e24cd8a523e360eef23143 2024-11-17T01:54:44,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/5c758def34c04bbba9e479943a8b4700 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/5c758def34c04bbba9e479943a8b4700 2024-11-17T01:54:44,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/66458c026bb04f2db5facc397d55a136 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/66458c026bb04f2db5facc397d55a136 2024-11-17T01:54:44,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/cba06c5daeb44be6a3f72651e97571bd to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/cba06c5daeb44be6a3f72651e97571bd 2024-11-17T01:54:44,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/43c1c2d9e7df413793d0da7fb010a8e0 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/43c1c2d9e7df413793d0da7fb010a8e0 2024-11-17T01:54:44,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/2c11150aba694de1a8ca5906db6b4465, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/0264caa5d94542299980416d1da0600e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/9368126a778645b1adaa15cd37ba0e18, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/e89e6a6ffbbc4e2f8237dd6525c52369, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/45f5c60a367a472ea0db53d99155343d, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/5d836060fd9e46cba1ef9fc7558592ed, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/9ad15290ede44f8782d62aa129700f34, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/1133af9c4dba4ad18742a84f8ad85370, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/679bde97850a48b7b6f63cb62b14de74, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/9805a8bc10264701a981363ef8662de7, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/c2eaf1e2135249cfa1a944b8be502425, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/2d3353a6b92445f38edc665908d8af24, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/bbb3631e6cd34afeaa8c62affad6990c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/59a76b27bfb94f2cbe94ff05bb2e0019, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/0abfb768b7c94a0d8f2f93ca8ac3ddeb, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/ccd85ff9674c4dd98856e251ccb86c2f, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/1e84cd0b4d674736a468d71c2caa8774, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/e1639ee8a4e74cfc853421d4bfd61d96] to archive 2024-11-17T01:54:44,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T01:54:44,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/2c11150aba694de1a8ca5906db6b4465 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/2c11150aba694de1a8ca5906db6b4465 2024-11-17T01:54:44,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/0264caa5d94542299980416d1da0600e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/0264caa5d94542299980416d1da0600e 2024-11-17T01:54:44,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/9368126a778645b1adaa15cd37ba0e18 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/9368126a778645b1adaa15cd37ba0e18 2024-11-17T01:54:44,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/e89e6a6ffbbc4e2f8237dd6525c52369 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/e89e6a6ffbbc4e2f8237dd6525c52369 2024-11-17T01:54:44,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/45f5c60a367a472ea0db53d99155343d to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/45f5c60a367a472ea0db53d99155343d 2024-11-17T01:54:44,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/5d836060fd9e46cba1ef9fc7558592ed to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/5d836060fd9e46cba1ef9fc7558592ed 2024-11-17T01:54:44,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/9ad15290ede44f8782d62aa129700f34 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/9ad15290ede44f8782d62aa129700f34 2024-11-17T01:54:44,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/1133af9c4dba4ad18742a84f8ad85370 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/1133af9c4dba4ad18742a84f8ad85370 2024-11-17T01:54:44,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/679bde97850a48b7b6f63cb62b14de74 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/679bde97850a48b7b6f63cb62b14de74 2024-11-17T01:54:44,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/9805a8bc10264701a981363ef8662de7 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/9805a8bc10264701a981363ef8662de7 2024-11-17T01:54:44,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/c2eaf1e2135249cfa1a944b8be502425 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/c2eaf1e2135249cfa1a944b8be502425 2024-11-17T01:54:44,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/2d3353a6b92445f38edc665908d8af24 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/2d3353a6b92445f38edc665908d8af24 2024-11-17T01:54:44,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/bbb3631e6cd34afeaa8c62affad6990c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/bbb3631e6cd34afeaa8c62affad6990c 2024-11-17T01:54:44,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/59a76b27bfb94f2cbe94ff05bb2e0019 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/59a76b27bfb94f2cbe94ff05bb2e0019 2024-11-17T01:54:44,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/0abfb768b7c94a0d8f2f93ca8ac3ddeb to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/0abfb768b7c94a0d8f2f93ca8ac3ddeb 2024-11-17T01:54:44,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/ccd85ff9674c4dd98856e251ccb86c2f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/ccd85ff9674c4dd98856e251ccb86c2f 2024-11-17T01:54:44,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/1e84cd0b4d674736a468d71c2caa8774 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/1e84cd0b4d674736a468d71c2caa8774 2024-11-17T01:54:44,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/e1639ee8a4e74cfc853421d4bfd61d96 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/e1639ee8a4e74cfc853421d4bfd61d96 2024-11-17T01:54:44,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/89c4639b336a494295a4ad3251bc20e5, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/05aac7bd4acf41f3be9bdb4970db148e, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/572312267a704bdcb2852597a475fea3, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/21d2360b268643c0a3e724fc9d37422a, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/a512c04aae9e4ef39e8386e0e3b23186, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/12e4824ce531494ab9b93bc2fba442f8, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/b4cbb3f998d44d528e07f89b96cc7fdf, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/aa04dffc9e844b269a1b51cb052e1b3c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/78efa55ba494483099e4e569e207bfe1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/634c48da330f4ee7a1c8380c0d670a79, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/a54b969ddc674191b2f494a47bb751c2, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/3b707741a0384b978963e9a690321277, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/56294fa1c5ea4ce08bff6625770db579, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/da32811eeed64f3fa49b521f813fcf06, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/9876425c602048cc909a73aaf7dffeab, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/92ca2f2b29b547aea292b90d6680332c, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/49515d0c8c544d28bee3320e5f0811e1, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/bdfddfa3a65e45d281d57366a3284159] to archive 2024-11-17T01:54:44,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T01:54:44,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/89c4639b336a494295a4ad3251bc20e5 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/89c4639b336a494295a4ad3251bc20e5 2024-11-17T01:54:44,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/05aac7bd4acf41f3be9bdb4970db148e to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/05aac7bd4acf41f3be9bdb4970db148e 2024-11-17T01:54:44,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/572312267a704bdcb2852597a475fea3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/572312267a704bdcb2852597a475fea3 2024-11-17T01:54:44,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/21d2360b268643c0a3e724fc9d37422a to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/21d2360b268643c0a3e724fc9d37422a 2024-11-17T01:54:44,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/a512c04aae9e4ef39e8386e0e3b23186 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/a512c04aae9e4ef39e8386e0e3b23186 2024-11-17T01:54:44,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/12e4824ce531494ab9b93bc2fba442f8 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/12e4824ce531494ab9b93bc2fba442f8 2024-11-17T01:54:44,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/b4cbb3f998d44d528e07f89b96cc7fdf to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/b4cbb3f998d44d528e07f89b96cc7fdf 2024-11-17T01:54:44,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/aa04dffc9e844b269a1b51cb052e1b3c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/aa04dffc9e844b269a1b51cb052e1b3c 2024-11-17T01:54:44,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/78efa55ba494483099e4e569e207bfe1 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/78efa55ba494483099e4e569e207bfe1 2024-11-17T01:54:44,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/634c48da330f4ee7a1c8380c0d670a79 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/634c48da330f4ee7a1c8380c0d670a79 2024-11-17T01:54:44,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/a54b969ddc674191b2f494a47bb751c2 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/a54b969ddc674191b2f494a47bb751c2 2024-11-17T01:54:44,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/3b707741a0384b978963e9a690321277 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/3b707741a0384b978963e9a690321277 2024-11-17T01:54:44,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/56294fa1c5ea4ce08bff6625770db579 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/56294fa1c5ea4ce08bff6625770db579 2024-11-17T01:54:44,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/da32811eeed64f3fa49b521f813fcf06 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/da32811eeed64f3fa49b521f813fcf06 2024-11-17T01:54:44,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/9876425c602048cc909a73aaf7dffeab to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/9876425c602048cc909a73aaf7dffeab 2024-11-17T01:54:44,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/92ca2f2b29b547aea292b90d6680332c to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/92ca2f2b29b547aea292b90d6680332c 2024-11-17T01:54:44,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/49515d0c8c544d28bee3320e5f0811e1 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/49515d0c8c544d28bee3320e5f0811e1 2024-11-17T01:54:44,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/bdfddfa3a65e45d281d57366a3284159 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/bdfddfa3a65e45d281d57366a3284159 2024-11-17T01:54:44,977 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/recovered.edits/267.seqid, newMaxSeqId=267, maxSeqId=4 2024-11-17T01:54:44,978 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. 2024-11-17T01:54:44,978 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for f6dcaac235b7e538d527b651e63f9dcc: Waiting for close lock at 1731808483225Running coprocessor pre-close hooks at 1731808483225Disabling compacts and flushes for region at 1731808483225Disabling writes for close at 1731808483226 (+1 ms)Obtaining lock to block concurrent updates at 1731808483226Preparing flush snapshotting stores in f6dcaac235b7e538d527b651e63f9dcc at 1731808483226Finished memstore snapshotting TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731808483227 (+1 ms)Flushing stores of TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc. at 1731808483228 (+1 ms)Flushing f6dcaac235b7e538d527b651e63f9dcc/A: creating writer at 1731808483228Flushing mob file f6dcaac235b7e538d527b651e63f9dcc/A: appending metadata at 1731808483237 (+9 ms)Flushing mob file f6dcaac235b7e538d527b651e63f9dcc/A: closing flushed file at 1731808483237Flushing f6dcaac235b7e538d527b651e63f9dcc/A: appending metadata at 1731808483644 (+407 ms)Flushing f6dcaac235b7e538d527b651e63f9dcc/A: closing flushed file at 1731808483644Flushing f6dcaac235b7e538d527b651e63f9dcc/B: creating writer at 1731808484055 (+411 ms)Flushing f6dcaac235b7e538d527b651e63f9dcc/B: appending metadata at 1731808484058 (+3 ms)Flushing f6dcaac235b7e538d527b651e63f9dcc/B: closing flushed file at 1731808484058Flushing f6dcaac235b7e538d527b651e63f9dcc/C: creating writer at 1731808484470 (+412 ms)Flushing f6dcaac235b7e538d527b651e63f9dcc/C: appending metadata at 1731808484475 (+5 ms)Flushing f6dcaac235b7e538d527b651e63f9dcc/C: closing flushed file at 1731808484475Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49909219: reopening flushed file at 1731808484887 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55727b7f: reopening flushed file at 1731808484894 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49c98053: reopening flushed file at 1731808484900 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for f6dcaac235b7e538d527b651e63f9dcc in 1680ms, sequenceid=264, compaction requested=false at 1731808484907 (+7 ms)Writing region close event to WAL at 1731808484974 (+67 ms)Running coprocessor post-close hooks at 1731808484978 (+4 ms)Closed at 1731808484978 2024-11-17T01:54:44,980 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:44,980 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=f6dcaac235b7e538d527b651e63f9dcc, regionState=CLOSED 2024-11-17T01:54:44,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 because future has completed 2024-11-17T01:54:44,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-17T01:54:44,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure f6dcaac235b7e538d527b651e63f9dcc, server=c6c0d7a0a7c0,41157,1731808292173 in 1.9120 sec 2024-11-17T01:54:44,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-17T01:54:44,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6dcaac235b7e538d527b651e63f9dcc, UNASSIGN in 1.9180 sec 2024-11-17T01:54:44,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-17T01:54:44,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9230 sec 2024-11-17T01:54:44,992 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808484992"}]},"ts":"1731808484992"} 2024-11-17T01:54:44,994 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T01:54:44,994 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T01:54:44,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9360 sec 2024-11-17T01:54:45,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T01:54:45,205 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:54:45,206 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T01:54:45,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:54:45,208 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:54:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T01:54:45,209 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:54:45,212 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:45,215 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A, FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B, FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C, FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/recovered.edits] 2024-11-17T01:54:45,218 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/0ed46852ba0a41809dd48a0df60cfca3 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/0ed46852ba0a41809dd48a0df60cfca3 2024-11-17T01:54:45,219 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/f50067bf51ed4903a86e0e763e4c610f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/A/f50067bf51ed4903a86e0e763e4c610f 2024-11-17T01:54:45,221 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/5f63df939cce44098fa4ca820f2fa287 to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/5f63df939cce44098fa4ca820f2fa287 2024-11-17T01:54:45,222 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/ee416d0bd89b43108dc03593d1e47cfa to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/B/ee416d0bd89b43108dc03593d1e47cfa 2024-11-17T01:54:45,224 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/38fe0a33b473446f99fbddf2f173c33f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/38fe0a33b473446f99fbddf2f173c33f 2024-11-17T01:54:45,225 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/45be18af0a664cd6a498c7302afc5d9f to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/C/45be18af0a664cd6a498c7302afc5d9f 2024-11-17T01:54:45,228 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/recovered.edits/267.seqid to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc/recovered.edits/267.seqid 2024-11-17T01:54:45,229 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/default/TestAcidGuarantees/f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:45,229 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T01:54:45,229 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T01:54:45,230 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T01:54:45,234 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111709a6ab44f60c46408ce126002ce826c6_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111709a6ab44f60c46408ce126002ce826c6_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:45,236 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172e9d9d165cbc49c289a7d8694890e139_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172e9d9d165cbc49c289a7d8694890e139_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:45,237 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173916334a3030468794f74380c8078637_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173916334a3030468794f74380c8078637_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:45,238 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173a64620444b64e05bb19a9cbe45a471d_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173a64620444b64e05bb19a9cbe45a471d_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:45,240 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117631d8c70dc66405691cf1ac617a7a5bd_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117631d8c70dc66405691cf1ac617a7a5bd_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:45,242 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117685fe40487a04ff1af23750ed65c9ac4_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117685fe40487a04ff1af23750ed65c9ac4_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:45,244 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177121e1e391ed43138b822b57c9c1df28_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177121e1e391ed43138b822b57c9c1df28_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:45,245 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111780113dfb03f94a80b31787fcbc53cb16_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111780113dfb03f94a80b31787fcbc53cb16_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:45,247 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178e1a1ecdc8d54b57b1ab6986b23c7496_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178e1a1ecdc8d54b57b1ab6986b23c7496_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:45,249 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117994b69b6c4774e26abd6966c33562dfa_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117994b69b6c4774e26abd6966c33562dfa_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:45,251 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b14e341792e74c1c8b27a24aaf8fabe5_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b14e341792e74c1c8b27a24aaf8fabe5_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:45,253 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f48fa1b739dd4adfa4dbacf03ff0c6bc_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f48fa1b739dd4adfa4dbacf03ff0c6bc_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:45,255 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fa270865afbc4eb5aaf0fe0594b0b786_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fa270865afbc4eb5aaf0fe0594b0b786_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:45,257 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fe9f16122b9a48ed81ce09ef719b235f_f6dcaac235b7e538d527b651e63f9dcc to hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fe9f16122b9a48ed81ce09ef719b235f_f6dcaac235b7e538d527b651e63f9dcc 2024-11-17T01:54:45,258 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T01:54:45,262 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:54:45,266 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T01:54:45,269 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T01:54:45,271 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:54:45,271 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T01:54:45,271 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731808485271"}]},"ts":"9223372036854775807"} 2024-11-17T01:54:45,274 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T01:54:45,274 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f6dcaac235b7e538d527b651e63f9dcc, NAME => 'TestAcidGuarantees,,1731808451119.f6dcaac235b7e538d527b651e63f9dcc.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T01:54:45,275 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T01:54:45,275 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731808485275"}]},"ts":"9223372036854775807"} 2024-11-17T01:54:45,276 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T01:54:45,277 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T01:54:45,278 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 71 msec 2024-11-17T01:54:45,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T01:54:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T01:54:45,315 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T01:54:45,324 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 251), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=372 (was 454), ProcessCount=11 (was 11), AvailableMemoryMB=5219 (was 5467) 2024-11-17T01:54:45,324 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-17T01:54:45,324 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T01:54:45,324 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:54:45,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:45,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:45,324 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:54:45,324 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-17T01:54:45,324 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1219616701, stopped=false 2024-11-17T01:54:45,325 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c6c0d7a0a7c0,45023,1731808291372 2024-11-17T01:54:45,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T01:54:45,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41157-0x10147c6f9be0001, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T01:54:45,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:45,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41157-0x10147c6f9be0001, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:45,375 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T01:54:45,376 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T01:54:45,376 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T01:54:45,376 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41157-0x10147c6f9be0001, quorum=127.0.0.1:59270, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T01:54:45,376 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:54:45,376 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:45,378 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c6c0d7a0a7c0,41157,1731808292173' ***** 2024-11-17T01:54:45,378 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T01:54:45,379 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T01:54:45,379 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T01:54:45,379 INFO [RS:0;c6c0d7a0a7c0:41157 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T01:54:45,380 INFO [RS:0;c6c0d7a0a7c0:41157 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T01:54:45,380 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(959): stopping server c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:45,380 INFO [RS:0;c6c0d7a0a7c0:41157 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T01:54:45,380 INFO [RS:0;c6c0d7a0a7c0:41157 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c6c0d7a0a7c0:41157. 2024-11-17T01:54:45,380 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:54:45,380 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:45,381 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T01:54:45,381 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T01:54:45,381 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T01:54:45,381 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-17T01:54:45,382 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-17T01:54:45,382 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-17T01:54:45,382 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T01:54:45,382 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T01:54:45,382 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T01:54:45,382 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T01:54:45,382 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T01:54:45,382 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T01:54:45,382 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-17T01:54:45,404 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/.tmp/info/c3221c330ba8480a9d6e3800178c94fd is 95, key is TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8./info:/1731808330253/DeleteFamily/seqid=0 2024-11-17T01:54:45,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742435_1611 (size=6514) 2024-11-17T01:54:45,430 INFO [regionserver/c6c0d7a0a7c0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T01:54:45,582 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T01:54:45,783 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T01:54:45,808 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/.tmp/info/c3221c330ba8480a9d6e3800178c94fd 2024-11-17T01:54:45,833 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/.tmp/ns/55b85a8defb940b787554c82e0233913 is 93, key is TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8./ns:/1731808330213/DeleteFamily/seqid=0 2024-11-17T01:54:45,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742436_1612 (size=5906) 2024-11-17T01:54:45,983 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T01:54:46,183 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T01:54:46,211 INFO [regionserver/c6c0d7a0a7c0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-17T01:54:46,211 INFO [regionserver/c6c0d7a0a7c0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-17T01:54:46,237 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/.tmp/ns/55b85a8defb940b787554c82e0233913 2024-11-17T01:54:46,259 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/.tmp/rep_barrier/709e2ad135924bed8bb335013617a5cf is 102, key is TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8./rep_barrier:/1731808330213/DeleteFamily/seqid=0 2024-11-17T01:54:46,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742437_1613 (size=6025) 2024-11-17T01:54:46,384 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T01:54:46,584 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T01:54:46,663 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/.tmp/rep_barrier/709e2ad135924bed8bb335013617a5cf 2024-11-17T01:54:46,684 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/.tmp/table/812000f961cd42f6b5507b57ceff16bb is 96, key is TestAcidGuarantees,,1731808296901.5d8a196529c6f40b7321f17bb79ff0a8./table:/1731808330213/DeleteFamily/seqid=0 2024-11-17T01:54:46,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742438_1614 (size=5936) 2024-11-17T01:54:46,784 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T01:54:46,984 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T01:54:47,088 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/.tmp/table/812000f961cd42f6b5507b57ceff16bb 2024-11-17T01:54:47,091 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/.tmp/info/c3221c330ba8480a9d6e3800178c94fd as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/info/c3221c330ba8480a9d6e3800178c94fd 2024-11-17T01:54:47,094 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/info/c3221c330ba8480a9d6e3800178c94fd, entries=12, sequenceid=90, filesize=6.4 K 2024-11-17T01:54:47,094 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/.tmp/ns/55b85a8defb940b787554c82e0233913 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/ns/55b85a8defb940b787554c82e0233913 2024-11-17T01:54:47,097 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/ns/55b85a8defb940b787554c82e0233913, entries=8, sequenceid=90, filesize=5.8 K 2024-11-17T01:54:47,098 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/.tmp/rep_barrier/709e2ad135924bed8bb335013617a5cf as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/rep_barrier/709e2ad135924bed8bb335013617a5cf 2024-11-17T01:54:47,100 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/rep_barrier/709e2ad135924bed8bb335013617a5cf, entries=6, sequenceid=90, filesize=5.9 K 2024-11-17T01:54:47,100 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/.tmp/table/812000f961cd42f6b5507b57ceff16bb as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/table/812000f961cd42f6b5507b57ceff16bb 2024-11-17T01:54:47,103 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/table/812000f961cd42f6b5507b57ceff16bb, entries=7, sequenceid=90, filesize=5.8 K 2024-11-17T01:54:47,103 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1721ms, sequenceid=90, compaction requested=false 2024-11-17T01:54:47,107 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-17T01:54:47,108 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T01:54:47,108 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T01:54:47,108 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731808485382Running coprocessor pre-close hooks at 1731808485382Disabling compacts and flushes for region at 1731808485382Disabling writes for close at 1731808485382Obtaining lock to block concurrent updates at 1731808485382Preparing flush snapshotting stores in 1588230740 at 1731808485382Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731808485383 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731808485384 (+1 ms)Flushing 1588230740/info: creating writer at 1731808485384Flushing 1588230740/info: appending metadata at 1731808485404 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731808485404Flushing 1588230740/ns: creating writer at 1731808485814 (+410 ms)Flushing 1588230740/ns: appending metadata at 1731808485833 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731808485833Flushing 1588230740/rep_barrier: creating writer at 1731808486242 (+409 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731808486259 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731808486259Flushing 1588230740/table: creating writer at 1731808486667 (+408 ms)Flushing 1588230740/table: appending metadata at 1731808486684 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731808486684Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7df73e36: reopening flushed file at 1731808487090 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38c89327: reopening flushed file at 1731808487094 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7afe9b64: reopening flushed file at 1731808487097 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f2c3302: reopening flushed file at 1731808487100 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1721ms, sequenceid=90, compaction requested=false at 1731808487103 (+3 ms)Writing region close event to WAL at 1731808487105 (+2 ms)Running coprocessor post-close hooks at 1731808487107 (+2 ms)Closed at 1731808487108 (+1 ms) 2024-11-17T01:54:47,108 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-17T01:54:47,185 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(976): stopping server c6c0d7a0a7c0,41157,1731808292173; all regions closed. 2024-11-17T01:54:47,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741834_1010 (size=25344) 2024-11-17T01:54:47,191 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/oldWALs 2024-11-17T01:54:47,191 INFO [RS:0;c6c0d7a0a7c0:41157 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c6c0d7a0a7c0%2C41157%2C1731808292173.meta:.meta(num 1731808296328) 2024-11-17T01:54:47,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741832_1008 (size=14332622) 2024-11-17T01:54:47,194 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/oldWALs 2024-11-17T01:54:47,194 INFO [RS:0;c6c0d7a0a7c0:41157 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c6c0d7a0a7c0%2C41157%2C1731808292173:(num 1731808295366) 2024-11-17T01:54:47,194 DEBUG [RS:0;c6c0d7a0a7c0:41157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:54:47,194 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T01:54:47,194 INFO [RS:0;c6c0d7a0a7c0:41157 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T01:54:47,194 INFO [RS:0;c6c0d7a0a7c0:41157 {}] hbase.ChoreService(370): Chore service for: regionserver/c6c0d7a0a7c0:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-17T01:54:47,195 INFO [RS:0;c6c0d7a0a7c0:41157 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T01:54:47,195 INFO [regionserver/c6c0d7a0a7c0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T01:54:47,195 INFO [RS:0;c6c0d7a0a7c0:41157 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41157 2024-11-17T01:54:47,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41157-0x10147c6f9be0001, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c6c0d7a0a7c0,41157,1731808292173 2024-11-17T01:54:47,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T01:54:47,260 INFO [RS:0;c6c0d7a0a7c0:41157 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T01:54:47,274 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c6c0d7a0a7c0,41157,1731808292173] 2024-11-17T01:54:47,282 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c6c0d7a0a7c0,41157,1731808292173 already deleted, retry=false 2024-11-17T01:54:47,282 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c6c0d7a0a7c0,41157,1731808292173 expired; onlineServers=0 2024-11-17T01:54:47,282 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c6c0d7a0a7c0,45023,1731808291372' ***** 2024-11-17T01:54:47,282 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-17T01:54:47,282 INFO [M:0;c6c0d7a0a7c0:45023 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T01:54:47,282 INFO [M:0;c6c0d7a0a7c0:45023 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T01:54:47,282 DEBUG [M:0;c6c0d7a0a7c0:45023 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-17T01:54:47,283 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-17T01:54:47,283 DEBUG [M:0;c6c0d7a0a7c0:45023 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-17T01:54:47,283 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.large.0-1731808295013 {}] cleaner.HFileCleaner(306): Exit Thread[master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.large.0-1731808295013,5,FailOnTimeoutGroup] 2024-11-17T01:54:47,283 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.small.0-1731808295014 {}] cleaner.HFileCleaner(306): Exit Thread[master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.small.0-1731808295014,5,FailOnTimeoutGroup] 2024-11-17T01:54:47,283 INFO [M:0;c6c0d7a0a7c0:45023 {}] hbase.ChoreService(370): Chore service for: master/c6c0d7a0a7c0:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-17T01:54:47,283 INFO [M:0;c6c0d7a0a7c0:45023 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T01:54:47,283 DEBUG [M:0;c6c0d7a0a7c0:45023 {}] master.HMaster(1795): Stopping service threads 2024-11-17T01:54:47,283 INFO [M:0;c6c0d7a0a7c0:45023 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-17T01:54:47,284 INFO [M:0;c6c0d7a0a7c0:45023 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T01:54:47,284 ERROR [M:0;c6c0d7a0a7c0:45023 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:35241 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:35241,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-17T01:54:47,285 INFO [M:0;c6c0d7a0a7c0:45023 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T01:54:47,285 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-17T01:54:47,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-17T01:54:47,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:54:47,291 DEBUG [M:0;c6c0d7a0a7c0:45023 {}] zookeeper.ZKUtil(347): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-17T01:54:47,291 WARN [M:0;c6c0d7a0a7c0:45023 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-17T01:54:47,291 INFO [M:0;c6c0d7a0a7c0:45023 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/.lastflushedseqids 2024-11-17T01:54:47,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742439_1615 (size=72) 2024-11-17T01:54:47,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41157-0x10147c6f9be0001, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:54:47,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41157-0x10147c6f9be0001, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:54:47,375 INFO [RS:0;c6c0d7a0a7c0:41157 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T01:54:47,376 INFO [RS:0;c6c0d7a0a7c0:41157 {}] regionserver.HRegionServer(1031): Exiting; stopping=c6c0d7a0a7c0,41157,1731808292173; zookeeper connection closed. 2024-11-17T01:54:47,376 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4a9d691a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4a9d691a 2024-11-17T01:54:47,377 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-17T01:54:47,705 INFO [M:0;c6c0d7a0a7c0:45023 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T01:54:47,705 INFO [M:0;c6c0d7a0a7c0:45023 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-17T01:54:47,706 DEBUG [M:0;c6c0d7a0a7c0:45023 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T01:54:47,706 INFO [M:0;c6c0d7a0a7c0:45023 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:54:47,706 DEBUG [M:0;c6c0d7a0a7c0:45023 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:54:47,706 DEBUG [M:0;c6c0d7a0a7c0:45023 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T01:54:47,706 DEBUG [M:0;c6c0d7a0a7c0:45023 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:54:47,706 INFO [M:0;c6c0d7a0a7c0:45023 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=775.62 KB heapSize=950.63 KB 2024-11-17T01:54:47,725 DEBUG [M:0;c6c0d7a0a7c0:45023 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aba0482ff9bd4419b0398f0b330dc3ea is 82, key is hbase:meta,,1/info:regioninfo/1731808296420/Put/seqid=0 2024-11-17T01:54:47,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742440_1616 (size=5672) 2024-11-17T01:54:48,129 INFO [M:0;c6c0d7a0a7c0:45023 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2164 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aba0482ff9bd4419b0398f0b330dc3ea 2024-11-17T01:54:48,154 DEBUG [M:0;c6c0d7a0a7c0:45023 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b96f9e52647848138b36947d3d7a6d22 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x91/proc:d/1731808454218/Put/seqid=0 2024-11-17T01:54:48,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742441_1617 (size=44654) 2024-11-17T01:54:48,558 INFO [M:0;c6c0d7a0a7c0:45023 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=775.06 KB at sequenceid=2164 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b96f9e52647848138b36947d3d7a6d22 2024-11-17T01:54:48,565 INFO [M:0;c6c0d7a0a7c0:45023 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b96f9e52647848138b36947d3d7a6d22 2024-11-17T01:54:48,584 DEBUG [M:0;c6c0d7a0a7c0:45023 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f67512fe211442debba983afefe28be7 is 69, key is c6c0d7a0a7c0,41157,1731808292173/rs:state/1731808295120/Put/seqid=0 2024-11-17T01:54:48,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073742442_1618 (size=5156) 2024-11-17T01:54:48,989 INFO [M:0;c6c0d7a0a7c0:45023 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2164 (bloomFilter=true), to=hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f67512fe211442debba983afefe28be7 2024-11-17T01:54:48,999 DEBUG [M:0;c6c0d7a0a7c0:45023 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aba0482ff9bd4419b0398f0b330dc3ea as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aba0482ff9bd4419b0398f0b330dc3ea 2024-11-17T01:54:49,003 INFO [M:0;c6c0d7a0a7c0:45023 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aba0482ff9bd4419b0398f0b330dc3ea, entries=8, sequenceid=2164, filesize=5.5 K 2024-11-17T01:54:49,004 DEBUG [M:0;c6c0d7a0a7c0:45023 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b96f9e52647848138b36947d3d7a6d22 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b96f9e52647848138b36947d3d7a6d22 2024-11-17T01:54:49,008 INFO [M:0;c6c0d7a0a7c0:45023 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b96f9e52647848138b36947d3d7a6d22 2024-11-17T01:54:49,008 INFO [M:0;c6c0d7a0a7c0:45023 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b96f9e52647848138b36947d3d7a6d22, entries=166, sequenceid=2164, filesize=43.6 K 2024-11-17T01:54:49,009 DEBUG [M:0;c6c0d7a0a7c0:45023 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f67512fe211442debba983afefe28be7 as hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f67512fe211442debba983afefe28be7 2024-11-17T01:54:49,012 INFO [M:0;c6c0d7a0a7c0:45023 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35241/user/jenkins/test-data/6be29f03-3efc-d409-0c80-8ff32a499fb1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f67512fe211442debba983afefe28be7, entries=1, sequenceid=2164, filesize=5.0 K 2024-11-17T01:54:49,013 INFO [M:0;c6c0d7a0a7c0:45023 {}] regionserver.HRegion(3140): Finished flush of dataSize ~775.62 KB/794233, heapSize ~950.34 KB/973144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1307ms, sequenceid=2164, compaction requested=false 2024-11-17T01:54:49,015 INFO [M:0;c6c0d7a0a7c0:45023 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:54:49,015 DEBUG [M:0;c6c0d7a0a7c0:45023 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731808487706Disabling compacts and flushes for region at 1731808487706Disabling writes for close at 1731808487706Obtaining lock to block concurrent updates at 1731808487706Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731808487706Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=794233, getHeapSize=973384, getOffHeapSize=0, getCellsCount=2355 at 1731808487706Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731808487707 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731808487707Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731808487725 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731808487725Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731808488134 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731808488154 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731808488154Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731808488565 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731808488584 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731808488584Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79357607: reopening flushed file at 1731808488998 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ab5fc3a: reopening flushed file at 1731808489004 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@558f552e: reopening flushed file at 1731808489008 (+4 ms)Finished flush of dataSize ~775.62 KB/794233, heapSize ~950.34 KB/973144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1307ms, sequenceid=2164, compaction requested=false at 1731808489013 (+5 ms)Writing region close event to WAL at 1731808489015 (+2 ms)Closed at 1731808489015 2024-11-17T01:54:49,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45229 is added to blk_1073741830_1006 (size=936076) 2024-11-17T01:54:49,019 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T01:54:49,019 INFO [M:0;c6c0d7a0a7c0:45023 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-17T01:54:49,019 INFO [M:0;c6c0d7a0a7c0:45023 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45023 2024-11-17T01:54:49,019 INFO [M:0;c6c0d7a0a7c0:45023 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T01:54:49,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:54:49,174 INFO [M:0;c6c0d7a0a7c0:45023 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T01:54:49,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45023-0x10147c6f9be0000, quorum=127.0.0.1:59270, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:54:49,186 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T01:54:49,189 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T01:54:49,189 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T01:54:49,190 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T01:54:49,190 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/hadoop.log.dir/,STOPPED} 2024-11-17T01:54:49,194 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-17T01:54:49,194 WARN [BP-2119396547-172.17.0.2-1731808288007 heartbeating to localhost/127.0.0.1:35241 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-17T01:54:49,194 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-17T01:54:49,194 WARN [BP-2119396547-172.17.0.2-1731808288007 heartbeating to localhost/127.0.0.1:35241 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2119396547-172.17.0.2-1731808288007 (Datanode Uuid 6f20295a-4c5e-4b90-8d70-b1b119004aa5) service to localhost/127.0.0.1:35241 2024-11-17T01:54:49,198 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/cluster_97028d77-d23a-8219-e606-b0c441f49b52/data/data1/current/BP-2119396547-172.17.0.2-1731808288007 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T01:54:49,199 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/cluster_97028d77-d23a-8219-e606-b0c441f49b52/data/data2/current/BP-2119396547-172.17.0.2-1731808288007 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T01:54:49,199 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-17T01:54:49,206 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T01:54:49,207 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T01:54:49,207 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T01:54:49,207 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T01:54:49,207 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59831bbf-37b3-9053-7e95-46c70d902fb8/hadoop.log.dir/,STOPPED} 2024-11-17T01:54:49,222 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-17T01:54:49,339 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down